Adding le32-nacl build of pnacl tools.

toolchain_build/toolchain_build_pnacl.py

 #!/usr/bin/python
 # Copyright (c) 2013 The Native Client Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Recipes for PNaCl toolchain packages.
 
    Recipes consist of specially-structured dictionaries, with keys for package
    name, type, commands to execute, etc. The structure is documented in the
    PackageBuilder docstring in toolchain_main.py.
@@ skipping 124 matching lines @@
 def TripleIsX8664(t):
   return fnmatch.fnmatch(t, 'x86_64*')
 
 
 # Return a tuple (C compiler, C++ compiler) of the compilers to compile the host
 # toolchains
 def CompilersForHost(host):
   compiler = {
       # For now we only do native builds for linux and mac
       # treat 32-bit linux like a native build
-      'i686-linux': (CHROME_CLANG, CHROME_CLANGXX),
-      'x86_64-linux': (CHROME_CLANG, CHROME_CLANGXX),
-      'x86_64-apple-darwin': (CHROME_CLANG, CHROME_CLANGXX),
+      'i686-linux': (CHROME_CLANG, CHROME_CLANGXX, 'ar', 'ranlib'),
+      'x86_64-linux': (CHROME_CLANG, CHROME_CLANGXX, 'ar', 'ranlib'),
+      'x86_64-apple-darwin': (CHROME_CLANG, CHROME_CLANGXX, 'ar', 'ranlib'),
       # Windows build should work for native and cross
-      'i686-w64-mingw32': ('i686-w64-mingw32-gcc', 'i686-w64-mingw32-g++'),
+      'i686-w64-mingw32': (
+          'i686-w64-mingw32-gcc', 'i686-w64-mingw32-g++', 'ar', 'ranlib'),
       # TODO: add arm-hosted support
-      'i686-pc-cygwin': ('gcc', 'g++'),
+      'i686-pc-cygwin': ('gcc', 'g++', 'ar', 'ranlib'),
   }
+  if host == 'le32-nacl':
+    nacl_sdk = os.environ.get('NACL_SDK_ROOT')
+    assert nacl_sdk, 'NACL_SDK_ROOT not set'
+    pnacl_bin_dir = os.path.join(nacl_sdk, 'toolchain/linux_pnacl/bin')
+    glibc_bin_dir = os.path.join(nacl_sdk, 'toolchain/linux_x86_glibc/bin')
+    compiler.update({
+        'le32-nacl': (os.path.join(pnacl_bin_dir, 'pnacl-clang'),
+                      os.path.join(pnacl_bin_dir, 'pnacl-clang++'),
+                      os.path.join(pnacl_bin_dir, 'pnacl-ar'),
+                      os.path.join(pnacl_bin_dir, 'pnacl-ranlib')),
+    })
   return compiler[host]
 
 
 def GSDJoin(*args):
   return '_'.join([pynacl.gsd_storage.LegalizeName(arg) for arg in args])
 
 
-def ConfigureHostArchFlags(host, extra_cflags, options):
+def ConfigureHostArchFlags(host, extra_cflags, options, extra_configure=None):
   """ Return flags passed to LLVM and binutils configure for compilers and
   compile flags. """
   configure_args = []
   extra_cc_args = []
 
+  configure_args += options.extra_configure_args
+  if extra_configure is not None:
+    configure_args += extra_configure
+  if options.extra_cc_args is not None:
+    extra_cc_args += [options.extra_cc_args]
+
   native = pynacl.platform.PlatformTriple()
   is_cross = host != native
   if is_cross:
     if (pynacl.platform.IsLinux64() and
         fnmatch.fnmatch(host, '*-linux*')):
       # 64 bit linux can build 32 bit linux binaries while still being a native
       # build for our purposes. But it's not what config.guess will yield, so
       # use --build to force it and make sure things build correctly.
       configure_args.append('--build=' + host)
     else:
       configure_args.append('--host=' + host)
   if TripleIsLinux(host) and not TripleIsX8664(host):
     # Chrome clang defaults to 64-bit builds, even when run on 32-bit Linux.
-    extra_cc_args = ['-m32']
+    extra_cc_args += ['-m32']
 
   extra_cxx_args = list(extra_cc_args)
 
   if not options.gcc:
-    cc, cxx = CompilersForHost(host)
+    cc, cxx, ar, ranlib = CompilersForHost(host)
 
     configure_args.append('CC=' + ' '.join([cc] + extra_cc_args))
     configure_args.append('CXX=' + ' '.join([cxx] + extra_cxx_args))
+    configure_args.append('AR=' + ar)
+    configure_args.append('RANLIB=' + ranlib)
 
   if TripleIsWindows(host):
     # The i18n support brings in runtime dependencies on MinGW DLLs
     # that we don't want to have to distribute alongside our binaries.
     # So just disable it, and compiler messages will always be in US English.
     configure_args.append('--disable-nls')
     configure_args.extend(['LDFLAGS=-L%(abs_libdl)s -ldl',
                            'CFLAGS=-isystem %(abs_libdl)s',
                            'CXXFLAGS=-isystem %(abs_libdl)s'])
     if is_cross:
       # LLVM's linux->mingw cross build needs this
       configure_args.append('CC_FOR_BUILD=gcc')
   else:
     if TripleIsMac(host):
       # This is required for building with recent libc++ against OSX 10.6
       extra_cflags.append('-U__STRICT_ANSI__')
-    if options.gcc:
+    if options.gcc or host == 'le32-nacl':
       configure_args.extend(['CFLAGS=' + ' '.join(extra_cflags),
                              'CXXFLAGS=' + ' '.join(extra_cflags)])
     else:
       configure_args.extend(
        ['CFLAGS=' + ' '.join(extra_cflags),
         'LDFLAGS=-L%(' + GSDJoin('abs_libcxx', host) + ')s/lib',
         'CXXFLAGS=-stdlib=libc++ -I%(' + GSDJoin('abs_libcxx', host) +
         ')s/include/c++/v1 ' + ' '.join(extra_cflags)])
 
   return configure_args
 
 
 def LibCxxHostArchFlags(host):
-  cc, cxx = CompilersForHost(host)
+  cc, cxx, _, _ = CompilersForHost(host)
   cmake_flags = []
   cmake_flags.extend(['-DCMAKE_C_COMPILER='+cc, '-DCMAKE_CXX_COMPILER='+cxx])
   if TripleIsLinux(host) and not TripleIsX8664(host):
     # Chrome clang defaults to 64-bit builds, even when run on 32-bit Linux
     cmake_flags.extend(['-DCMAKE_C_FLAGS=-m32',
                         '-DCMAKE_CXX_FLAGS=-m32'])
   return cmake_flags
 
 def CmakeHostArchFlags(host, options):
   """ Set flags passed to LLVM cmake for compilers and compile flags. """
   cmake_flags = []
-  cc, cxx = CompilersForHost(host)
+  cc, cxx, _, _ = CompilersForHost(host)
 
   cmake_flags.extend(['-DCMAKE_C_COMPILER='+cc, '-DCMAKE_CXX_COMPILER='+cxx])
 
   # There seems to be a bug in chrome clang where it exposes the msan interface
   # (even when compiling without msan) but then does not link with an
   # msan-enabled compiler_rt, leaving references to __msan_allocated_memory
   # undefined.
   cmake_flags.append('-DHAVE_SANITIZER_MSAN_INTERFACE_H=FALSE')
 
   if options.sanitize:
@@ skipping 143 matching lines @@
   if TripleIsLinux(host):
     libname = 'libc++.so.1'
   elif TripleIsMac(host):
     libname = 'libc++.1.dylib'
   else:
     return []
   return [command.Mkdir(dest, parents=True),
           command.Copy('%(' + GSDJoin('abs_libcxx', host) +')s/lib/' + libname,
                        os.path.join(dest, libname))]
 
-def CreateSymLinksToDirectToNaClTools():
+def CreateSymLinksToDirectToNaClTools(host):
+  if host == 'le32-nacl':
+    return []
   return (
       [command.Command(['ln', '-f',
                         command.path.join('%(output)s', 'bin','clang'),
                         command.path.join('%(output)s', 'bin',
                                           arch + '-nacl-clang')])
        for arch in DIRECT_TO_NACL_ARCHES] +
       [command.Command(['ln', '-f',
                         command.path.join('%(output)s', 'bin','clang'),
                         command.path.join('%(output)s', 'bin',
                                           arch + '-nacl-clang++')])
@@ skipping 58 matching lines @@
 def HostTools(host, options):
   def H(component_name):
     # Return a package name for a component name with a host triple.
     return GSDJoin(component_name, host)
   # Return the file name with the appropriate suffix for an executable file.
   def Exe(file):
     if TripleIsWindows(host):
       return file + '.exe'
     else:
       return file
+
+  werror = []
+  extra_gold_deps = []
+  if host == 'le32-nacl':
+    # TODO(bradnelson): Fix warnings so this can go away.
+    werror = ['--enable-werror=no']
+    extra_gold_deps = [H('llvm')]
+
   # Binutils still has some warnings when building with clang
   if not options.gcc:
     warning_flags = ['-Wno-extended-offsetof', '-Wno-absolute-value',
                     '-Wno-unused-function', '-Wno-unused-const-variable',
                     '-Wno-unneeded-internal-declaration',
                     '-Wno-unused-private-field', '-Wno-format-security']
   else:
     warning_flags = ['-Wno-unused-function', '-Wno-unused-value']
+
   tools = {
       # The binutils_pnacl package is used both for bitcode linking (gold) and
       # for its conventional use with arm-nacl-clang.
       H('binutils_pnacl'): {
-          'dependencies': ['binutils_pnacl_src'],
+          'dependencies': ['binutils_pnacl_src'] + extra_gold_deps,
           'type': 'build',
           'inputs' : { 'macros': os.path.join(NACL_DIR,
               'pnacl', 'support', 'clang_direct', 'nacl-arm-macros.s')},
           'commands': [
               command.SkipForIncrementalCommand([
                   'sh',
                   '%(binutils_pnacl_src)s/configure'] +
                   ConfigureBinutilsCommon() +
-                  ConfigureHostArchFlags(host, warning_flags, options) +
+                  ConfigureHostArchFlags(
+                    host, warning_flags, options,
+                    options.binutils_pnacl_extra_configure) +
                   ['--target=arm-nacl',
                   '--program-prefix=le32-nacl-',
                   '--enable-targets=arm-nacl,i686-nacl,x86_64-nacl,' +
                   'mipsel-nacl',
                   '--enable-shared=no',
                   '--enable-gold=default',
                   '--enable-plugins',
                   '--without-gas',
-                  '--with-sysroot=/le32-nacl']),
+                  '--with-sysroot=/le32-nacl'] + werror),
               command.Command(MakeCommand(host)),
               command.Command(MAKE_DESTDIR_CMD + ['install-strip'])] +
               [command.RemoveDirectory(os.path.join('%(output)s', dir))
                for dir in ('lib', 'lib32')] +
               # Since it has dual use, just create links for both sets of names
               # (i.e. le32-nacl-foo and arm-nacl-foo)
               # TODO(dschuff): Use the redirector scripts here like binutils_x86
               [command.Command([
                   'ln', '-f',
                   command.path.join('%(output)s', 'bin', 'le32-nacl-' + tool),
@@ skipping 44 matching lines @@
                   '-DLLVM_ENABLE_ASSERTIONS=ON',
                   '-DLLVM_ENABLE_ZLIB=OFF',
                   '-DLLVM_BUILD_TESTS=ON',
                   '-DLLVM_APPEND_VC_REV=ON',
                   '-DLLVM_BINUTILS_INCDIR=%(abs_binutils_pnacl_src)s/include',
                   '-DLLVM_EXTERNAL_CLANG_SOURCE_DIR=%(clang_src)s',
                   '%(llvm_src)s']),
               command.Command(['ninja', '-v']),
               command.Command(['ninja', 'install']),
               ] +
-          CreateSymLinksToDirectToNaClTools()
+          CreateSymLinksToDirectToNaClTools(host)
       },
   }
+  cleanup_static_libs = []
+  shared = []
+  if host != 'le32-nacl':
+    shared = ['--enable-shared']
+    cleanup_static_libs = [
+        command.Remove(*[os.path.join('%(output)s', 'lib', f) for f
+                         in '*.a', '*Hello.*', 'BugpointPasses.*']),
+    ]
   llvm_autoconf = {
       H('llvm'): {
           'dependencies': ['clang_src', 'llvm_src', 'binutils_pnacl_src',
                            'subzero_src'],
           'type': 'build',
           'commands': [
               command.SkipForIncrementalCommand([
                   'sh',
                   '%(llvm_src)s/configure'] +
                   ConfigureHostArchFlags(host, [], options) +
                   LLVMConfigureAssertionsFlags(options) +
                   ['--prefix=/',
-                   '--enable-shared',
                    '--disable-zlib',
                    '--disable-terminfo',
                    '--disable-jit',
                    '--disable-bindings', # ocaml is currently the only binding.
                    '--with-binutils-include=%(abs_binutils_pnacl_src)s/include',
                    '--enable-targets=x86,arm,mips',
                    '--program-prefix=',
                    '--enable-optimized',
-                   '--with-clang-srcdir=%(abs_clang_src)s'])] +
+                   '--with-clang-srcdir=%(abs_clang_src)s'] + shared)] +
               CopyHostLibcxxForLLVMBuild(
                   host,
                   os.path.join('Release+Asserts', 'lib'),
                   options) +
               [command.Command(MakeCommand(host) + [
                   'VERBOSE=1',
                   'NACL_SANDBOX=0',
+                  'PNACL_BROWSER_TRANSLATOR=0',
                   'SUBZERO_SRC_ROOT=%(abs_subzero_src)s',
                   'all']),
-              command.Command(MAKE_DESTDIR_CMD + ['install']),
-              command.Remove(*[os.path.join('%(output)s', 'lib', f) for f in
-                              '*.a', '*Hello.*', 'BugpointPasses.*']),
+              command.Command(MAKE_DESTDIR_CMD + ['install'])] +
+              cleanup_static_libs + [
               command.Remove(*[os.path.join('%(output)s', 'bin', f) for f in
                                Exe('clang-format'), Exe('clang-check'),
                                Exe('c-index-test'), Exe('clang-tblgen'),
                                Exe('llvm-tblgen')])] +
-              CreateSymLinksToDirectToNaClTools() +
+              CreateSymLinksToDirectToNaClTools(host) +
               CopyWindowsHostLibs(host),
       },
   }
   if options.cmake:
     tools.update(llvm_cmake)
   else:
     tools.update(llvm_autoconf)
   if TripleIsWindows(host):
     tools[H('binutils_pnacl')]['dependencies'].append('libdl')
     tools[H('llvm')]['dependencies'].append('libdl')
-  elif not options.gcc:
+  elif not options.gcc and host != 'le32-nacl':
     tools[H('binutils_pnacl')]['dependencies'].append(H('libcxx'))
     tools[H('llvm')]['dependencies'].append(H('libcxx'))
   return tools
 
 
 def TargetLibCompiler(host, options):
   def H(component_name):
     return GSDJoin(component_name, host)
   compiler = {
       # Because target_lib_compiler is not a memoized target, its name doesn't
@@ skipping 53 matching lines @@
 
   tools = {}
 
   if TripleIsWindows(host):
     redirector_table = ''
     for tool, args in TOOL_X64_I686_REDIRECTS:
       redirector_table += '  {L"/bin/i686-nacl-%s.exe",' % tool + \
                           '   L"/bin/x86_64-nacl-%s.exe",' % tool + \
                           ' L"%s"},\n' % args
 
-    cc, cxx = CompilersForHost(host)
+    cc, cxx, _, _ = CompilersForHost(host)
     tools.update({
         'redirector': {
             'type': 'build',
             'inputs': { 'source_directory': REDIRECTOR_WIN32_SRC },
             'commands': [
                 command.WriteData(redirector_table,
                                   'redirector_table_pnacl.txt'),
                 command.Command([cc, '-O3', '-std=c99', '-I.', '-o',
                                  os.path.join('%(output)s', 'redirector.exe'),
                                  '-I', os.path.dirname(NACL_DIR),
@@ skipping 236 matching lines @@
   parser.add_argument('--gcc', action='store_true', default=False,
                       help="Use the default compiler 'cc' instead of clang")
   parser.add_argument('--sanitize', choices=['address', 'thread', 'memory',
                                              'undefined'],
                       help="Use a sanitizer with LLVM's clang cmake build")
   parser.add_argument('--testsuite-sync', action='store_true', default=False,
                       help=('Sync the sources for the LLVM testsuite. '
                       'Only useful if --sync/ is also enabled'))
   parser.add_argument('--build-sbtc', action='store_true', default=False,
                       help='Build the sandboxed translators')
+  parser.add_argument('--pnacl-in-pnacl', action='store_true', default=False,
+                      help='Build with a PNaCl toolchain')
+  parser.add_argument('--extra-cc-args', default=None,
+                      help='Extra arguments to pass to cc/cxx')
+  parser.add_argument('--extra-configure-arg', dest='extra_configure_args',
+                      default=[], action='append',
+                      help='Extra arguments to pass pass to host configure')
+  parser.add_argument('--binutils-pnacl-extra-configure',
+                      default=[], action='append',
+                      help='Extra binutils-pnacl arguments '
+                           'to pass pass to host configure')
   args, leftover_args = parser.parse_known_args()
   if '-h' in leftover_args or '--help' in leftover_args:
     print 'The following arguments are specific to toolchain_build_pnacl.py:'
     parser.print_help()
     print 'The rest of the arguments are generic, in toolchain_main.py'
 
   if args.sanitize and not args.cmake:
     print 'Use of sanitizers requires a cmake build'
     sys.exit(1)
 
@@ skipping 13 matching lines @@
       leftover_args.append('--sync-only')
   else:
     upload_packages = GetUploadPackageTargets()
     if pynacl.platform.IsWindows():
       InstallMinGWHostCompiler()
 
     packages.update(HostToolsSources(GetGitSyncCmdsCallback(rev)))
     if args.testsuite_sync:
       packages.update(TestsuiteSources(GetGitSyncCmdsCallback(rev)))
 
-    hosts = [pynacl.platform.PlatformTriple()]
+    if args.pnacl_in_pnacl:
+      hosts = ['le32-nacl']
+    else:
+      hosts = [pynacl.platform.PlatformTriple()]
     if pynacl.platform.IsLinux() and BUILD_CROSS_MINGW:
       hosts.append(pynacl.platform.PlatformTriple('win', 'x86-32'))
     for host in hosts:
-      packages.update(HostLibs(host, args))
       packages.update(HostTools(host, args))
-      packages.update(HostToolsDirectToNacl(host))
-    packages.update(TargetLibCompiler(pynacl.platform.PlatformTriple(), args))
+      if not args.pnacl_in_pnacl:
+        packages.update(HostLibs(host, args))
+        packages.update(HostToolsDirectToNacl(host))
+    if not args.pnacl_in_pnacl:
+      packages.update(TargetLibCompiler(pynacl.platform.PlatformTriple(), args))
     # Don't build the target libs on Windows because of pathname issues.
     # Only the linux64 bot is canonical (i.e. it will upload its packages).
     # The other bots will use a 'work' target instead of a 'build' target for
     # the target libs, so they will not be memoized, but can be used for tests.
     # TODO(dschuff): Even better would be if we could memoize non-canonical
     # build targets without doing things like mangling their names (and for e.g.
     # scons tests, skip running them if their dependencies haven't changed, like
     # build targets)
     is_canonical = pynacl.platform.IsLinux64()
-    if pynacl.platform.IsLinux() or pynacl.platform.IsMac():
+    if ((pynacl.platform.IsLinux() or pynacl.platform.IsMac())
+        and not args.pnacl_in_pnacl):
       packages.update(pnacl_targetlibs.TargetLibsSrc(
         GetGitSyncCmdsCallback(rev)))
       for bias in BITCODE_BIASES:
         packages.update(pnacl_targetlibs.TargetLibs(bias, is_canonical))
       for arch in DIRECT_TO_NACL_ARCHES:
         packages.update(pnacl_targetlibs.TargetLibs(arch, is_canonical))
         packages.update(pnacl_targetlibs.SDKLibs(arch, is_canonical))
       for arch in TRANSLATOR_ARCHES:
         packages.update(pnacl_targetlibs.TranslatorLibs(arch, is_canonical))
       packages.update(Metadata(rev, is_canonical))
       packages.update(pnacl_targetlibs.SDKCompiler(
                       ['le32'] + DIRECT_TO_NACL_ARCHES))
       packages.update(pnacl_targetlibs.SDKLibs('le32', is_canonical))
-    if pynacl.platform.IsLinux() or pynacl.platform.IsMac():
       unsandboxed_irt_canonical = is_canonical or pynacl.platform.IsMac()
       packages.update(pnacl_targetlibs.UnsandboxedIRT(
           'x86-32-%s' % pynacl.platform.GetOS(), unsandboxed_irt_canonical))
 
-    if args.build_sbtc:
+    if args.build_sbtc and not args.pnacl_in_pnacl:
       packages.update(pnacl_sandboxed_translator.SandboxedTranslators(
         SANDBOXED_TRANSLATOR_ARCHES))
 
   tb = toolchain_main.PackageBuilder(packages,
                                      upload_packages,
                                      leftover_args)
   tb.Main()