toolchain_build: Build glibc-flavored gcc_libs and sdk_libs

toolchain_build/toolchain_build.py

 #!/usr/bin/python
 # Copyright (c) 2012 The Native Client Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Recipes for NativeClient toolchain packages.
 
 The real entry plumbing is in toolchain_main.py.
 """
 
@@ skipping 602 matching lines @@
       }
   return host_gcc_libs
 
 
 HOST_GCC_LIBS_DEPS = ['gmp', 'mpfr', 'mpc', 'isl', 'cloog']
 
 def HostGccLibsDeps(host):
   return [ForHost(package, host) for package in HOST_GCC_LIBS_DEPS]
 
 
-def SDKLibs(host, target):
+def SDKLibs(host, target, libc):
   def H(component_name):
     return ForHost(component_name, host)
-  components = ['newlib_%s' % target,
-                'gcc_libs_%s' % target,
-                H('binutils_%s' % target),
-                H('gcc_%s' % target),
-                ]
-  sdk_compiler = {
-    H('sdk_compiler_%s' % target): {
+
+  if libc == 'newlib':
+    libs_name = 'libs'
+  else:
+    libs_name = 'libs_' + libc
+
+  host_components = [H('binutils_%s' % target), H('gcc_%s' % target)]
+  target_components = [libc + '_' + target, 'gcc_' + libs_name + '_' + target]
+  components = host_components + target_components
+
+  sdk_compiler = H('sdk_compiler_' + libc + '_' + target)
+
+  builds = {
+    sdk_compiler: {
         'type': 'work',
         'dependencies': components,
         'commands': [command.CopyRecursive('%(' + item + ')s', '%(output)s')
                      for item in components],
     },
-  }
-  sdk_libs = {
-    'sdk_libs_%s' % target: {
+
+    'sdk_' + libs_name + '_' + target: {
         'type': 'build',
-        'dependencies': [H('sdk_compiler_%s' % target)],
+        'dependencies': [sdk_compiler],
         'inputs': {
             'src_untrusted': os.path.join(NACL_DIR, 'src', 'untrusted'),
             'src_include': os.path.join(NACL_DIR, 'src', 'include'),
             'scons.py': os.path.join(NACL_DIR, 'scons.py'),
             'site_scons': os.path.join(NACL_DIR, 'site_scons'),
         },
         'commands': [
           command.Command(
               [sys.executable, '%(scons.py)s',
-               '--verbose', 'MODE=nacl', '-j%(cores)s', 'naclsdk_validate=0',
+               '--verbose', '--mode=nacl', '-j%(cores)s', 'naclsdk_validate=0',
                'platform=%s' % target,
-               'nacl_newlib_dir=%(abs_' + H('sdk_compiler_%s' % target) + ')s',
+               'nacl_' + libc + '_dir=%(abs_' + sdk_compiler + ')s',
                'DESTINATION_ROOT=%(work_dir)s',
                'includedir=' + command.path.join('%(output)s',
                                                  target + '-nacl', 'include'),
                'libdir=' + command.path.join('%(output)s',
                                              target + '-nacl', 'lib'),
                'install'],
               cwd=NACL_DIR),
         ],
     },
   }
 
-  return dict(sdk_compiler.items() + sdk_libs.items())
+  return builds
 
 
 def ConfigureCommand(source_component):
   return [command % {'src': '%(' + source_component + ')s'}
           for command in CONFIGURE_CMD]
 
 
 # When doing a Canadian cross, we need native-hosted cross components
 # to do the GCC build.
 def GccDeps(host, target):
@@ skipping 18 matching lines @@
       ConfigureCommand(source_component) +
       ConfigureHostTool(host) +
       ConfigureTargetArgs(target) +
       TARGET_GCC_CONFIG.get(target, []) + [
           '--with-gmp=%(abs_' + ForHost('gmp', host) + ')s',
           '--with-mpfr=%(abs_' + ForHost('mpfr', host) + ')s',
           '--with-mpc=%(abs_' + ForHost('mpc', host) + ')s',
           '--with-isl=%(abs_' + ForHost('isl', host) + ')s',
           '--with-cloog=%(abs_' + ForHost('cloog', host) + ')s',
           '--enable-cloog-backend=isl',
-          '--disable-dlopen',
-          '--disable-shared',
-          '--with-newlib',
           '--with-linker-hash-style=gnu',
           '--enable-linker-build-id',
           '--enable-languages=c,c++,lto',
           ] + extra_args)
 
 
+GCC_NEWLIB = [

[Derek Schuff, 2015/04/23 19:48:09]

how about calling this GCC_NEWLIB_CONFIGURE or or GCC_CONFIGURE_FOR_NEWLIB or
something else slightly more descriptive?

+    '--disable-dlopen',
+    '--disable-shared',
+    '--with-newlib',
+    ]
+
 
 def HostTools(host, target):
   def H(component_name):
     return ForHost(component_name, host)
 
   def WindowsAlternate(if_windows, if_not_windows, if_mac=None):
     if if_mac is not None and HostIsMac(host):
       return if_mac
     elif HostIsWindows(host):
       return if_windows
@@ skipping 50 matching lines @@
               # that we don't want to include in the distribution.
               [command.RemoveDirectory(command.path.join('%(output)s', name))
                for name in ['lib', 'lib32', 'lib64']],
           },
 
       H('gcc_' + target): {
           'type': 'build',
           'dependencies': (['gcc'] + HostGccLibsDeps(host) +
                            GccDeps(host, target)),
           'commands': ConfigureTargetPrep(target) + [
-              ConfigureGccCommand('gcc', host, target),
+              ConfigureGccCommand('gcc', host, target, GCC_NEWLIB),
               # GCC's configure step writes configargs.h with some strings
               # including the configure command line, which get embedded
               # into the gcc driver binary.  The build only works if we use
               # absolute paths in some of the configure switches, but
               # embedding those paths makes the output differ in repeated
               # builds done in different directories, which we do not want.
               # So force the generation of that file early and then edit it
               # in place to replace the absolute paths with something that
               # never varies.  Note that the 'configure-gcc' target will
               # actually build some components before running gcc/configure.
@@ skipping 94 matching lines @@
   return commands
 
 
 def TargetLibs(host, target):
   lib_deps = [ForHost(component + '_' + target, host)
               for component in ['binutils', 'gcc']]
 
   def NewlibFile(subdir, name):
     return command.path.join('%(output)s', target + '-nacl', subdir, name)
 
-  newlib_sysroot = '%(abs_newlib_' + target + ')s'
-  newlib_tooldir = '%s/%s-nacl' % (newlib_sysroot, target)
-
   # See the comment at ConfigureTargetPrep, above.
   newlib_install_data = ' '.join(['STRIPPROG=%(cwd)s/strip_for_target',
                                   '%(abs_newlib)s/install-sh',
                                   '-c', '-s', '-m', '644'])
 
   iconv_encodings = 'UTF-8,UTF-16LE,UCS-4LE,UTF-16,UCS-4'
   newlib_configure_args = [
       '--disable-libgloss',
       '--enable-newlib-iconv',
       '--enable-newlib-iconv-from-encodings=' + iconv_encodings,
@@ skipping 11 matching lines @@
                      NewlibFile('lib', 'libcrt_common.a')),
       command.WriteData(NewlibLibcScript(target),
                         NewlibFile('lib', 'libc.a')),
       ] + [
       command.Copy(
           command.path.join('%(pthread_headers)s', header),
           NewlibFile('include', header))
       for header in ('pthread.h', 'semaphore.h')
       ]
 
+
+  def GccLibsTarget(libc, configure_args):
+    sysroot = '%(abs_' + libc + '_' + target + ')s'
+    tooldir = '%s/%s-nacl' % (sysroot, target)
+    package = {
+        'type': 'build',
+        'dependencies': (['gcc_libs'] + lib_deps + [libc + '_' + target] +
+                         HostGccLibsDeps(host)),
+        # This actually builds the compiler again and uses that compiler
+        # to build the target libraries.  That's by far the easiest thing
+        # to get going given the interdependencies of the target
+        # libraries (especially libgcc) on the gcc subdirectory, and
+        # building the compiler doesn't really take all that long in the
+        # grand scheme of things.
+        # TODO(mcgrathr): If upstream ever cleans up all their
+        # interdependencies better, unpack the compiler, configure with
+        # --disable-gcc, and just build all-target.
+        'commands': ConfigureTargetPrep(target) + [
+            ConfigureGccCommand('gcc_libs', host, target, configure_args + [
+                '--with-build-sysroot=' + tooldir,
+                ]),
+            GccCommand(host, target,
+                       MakeCommand(host) + [
+                           'build_tooldir=' + tooldir,
+                           'MAKEOVERRIDES=NATIVE_SYSTEM_HEADER_DIR=/include',
+                           'all-target',
+                           ]),
+            GccCommand(host, target,
+                       MAKE_DESTDIR_CMD + ['install-strip-target']),
+            REMOVE_INFO_DIR,
+            ],
+        }
+    return package
+
   # The 'minisdk_<target>' component is a workalike subset of what the full
   # NaCl SDK provides.  The glibc build uses a handful of things from the
   # SDK (ncval, sel_ldr, etc.), and expects them relative to $NACL_SDK_ROOT
   # in the layout that the SDK uses.  We provide a small subset built here
   # using SCons (and explicit copying, below), containing only the things
   # the build actually needs.
   def SconsCommand(args):
     return command.Command([sys.executable, '%(scons.py)s',
                             '--verbose', '-j%(cores)s',
                             'DESTINATION_ROOT=%(abs_work_dir)s'] + args,
@@ skipping 81 matching lines @@
                            ConfigureHostTool(host) +
                            ConfigureTargetArgs(target) +
                            newlib_configure_args,
                            MakeCommand(host),
                            MAKE_DESTDIR_CMD + ['install-strip'],
                            ]) +
                        newlib_post_install +
                        InstallDocFiles('newlib', ['COPYING.NEWLIB'])),
           },
 
+      'gcc_libs_' + target: GccLibsTarget('newlib', GCC_NEWLIB),
+
       'glibc_' + target: {
           'type': 'build',
           # The glibc build needs a libgcc.a (and the unwind.h header).
           # For now, we just use the one built for the newlib toolchain
           # and that seems to suffice.
           # TODO(mcgrathr): Eventually figure out a proper minimal build
           # of just the libgcc bits needed for the glibc build.
           'dependencies': ['glibc', 'minisdk_' + target,
                            'gcc_libs_' + target] + lib_deps,
           'commands': (ConfigureTargetPrep(target) +
@@ skipping 17 matching lines @@
                             'install_root=%(abs_output)s/' + target + '-nacl',
                             'inst_infodir=%(abs_output)s/share/info',
                            ],
                            ], target_deps=['gcc_libs']) +
                        InstallDocFiles('glibc', ['COPYING.LIB']) + [
                            REMOVE_INFO_DIR,
                        ]
                        ),
         },
 
-      'gcc_libs_' + target: {
-          'type': 'build',
-          'dependencies': (['gcc_libs'] + lib_deps + ['newlib_' + target] +
-                           HostGccLibsDeps(host)),
-          # This actually builds the compiler again and uses that compiler
-          # to build the target libraries.  That's by far the easiest thing
-          # to get going given the interdependencies of the target
-          # libraries (especially libgcc) on the gcc subdirectory, and
-          # building the compiler doesn't really take all that long in the
-          # grand scheme of things.
-          # TODO(mcgrathr): If upstream ever cleans up all their
-          # interdependencies better, unpack the compiler, configure with
-          # --disable-gcc, and just build all-target.
-          'commands': ConfigureTargetPrep(target) + [
-              ConfigureGccCommand('gcc_libs', host, target, [
-                  '--with-build-sysroot=' + newlib_sysroot,
-                  ]),
-              GccCommand(host, target,
-                         MakeCommand(host) + [
-                             'build_tooldir=' + newlib_tooldir,
-                             'all-target',
-                             ]),
-              GccCommand(host, target,
-                         MAKE_DESTDIR_CMD + ['install-strip-target']),
-              REMOVE_INFO_DIR,
-              ],
-          },
+      'gcc_libs_glibc_' + target: GccLibsTarget('glibc', []),
       }
 
   libs.update(support)
   return libs
 
 # Compute it once.
 NATIVE_TUPLE = pynacl.platform.PlatformTriple()
 
 
 # For our purposes, "cross-compiling" means not literally that we are
@@ skipping 53 matching lines @@
                                                        host_target.arch)
       binutils_package = ForHost('binutils_%s' % target_arch, platform_triple)
       gcc_package = ForHost('gcc_%s' % target_arch, platform_triple)
       gdb_package = ForHost('gdb', platform_triple)
 
       # Create a list of packages for a target.
       platform_packages = [binutils_package, gcc_package, gdb_package]
       raw_packages = shared_packages + platform_packages
       all_packages = raw_packages + sdk_lib_packages
 
-      glibc_packages = platform_packages + ['glibc_' + target_arch]
+      glibc_raw_packages = platform_packages + ['glibc_' + target_arch]
+      glibc_all_packages = glibc_raw_packages + ['sdk_libs_glibc_' +
+                                                 target_arch]
 
       os_name = pynacl.platform.GetOS(host_target.os)
       if host_target.differ3264:
         arch_name = pynacl.platform.GetArch3264(host_target.arch)
       else:
         arch_name = pynacl.platform.GetArch(host_target.arch)
       package_target = '%s_%s' % (os_name, arch_name)
       package_name = '%snacl_%s_newlib' % (package_prefix,
                                            pynacl.platform.GetArch(target_arch))
       raw_package_name = package_name + '_raw'
 
       # Toolchains by default are "raw" unless they include the Core SDK
       package_target_dict = package_targets.setdefault(package_target, {})
       package_target_dict.setdefault(raw_package_name, []).extend(raw_packages)
       package_target_dict.setdefault(package_name, []).extend(all_packages)
 
       glibc_package_name = (package_prefix +
                             ('nacl_%s_glibc' %
                              pynacl.platform.GetArch(target_arch)))
       glibc_raw_package_name = glibc_package_name + '_raw'
       package_target_dict.setdefault(glibc_raw_package_name,
-                                     []).extend(glibc_packages)
+                                     []).extend(glibc_raw_packages)
+      package_target_dict.setdefault(glibc_package_name,
+                                     []).extend(glibc_all_packages)
 
   # GDB is a special and shared, we will inject it into various other packages.
   for platform, arch in GDB_INJECT_HOSTS:
     platform_triple = pynacl.platform.PlatformTriple(platform, arch)
     os_name = pynacl.platform.GetOS(platform)
     arch_name = pynacl.platform.GetArch(arch)
 
     gdb_packages = [ForHost('gdb', platform_triple)]
     package_target = '%s_%s' % (os_name, arch_name)
 
     for package_name, package_archives in GDB_INJECT_PACKAGES:
       combined_packages = package_archives + gdb_packages
       package_target_dict = package_targets.setdefault(package_target, {})
       package_target_dict.setdefault(package_name, []).extend(combined_packages)
 
   return dict(package_targets)
 
 
 def CollectPackagesForHost(host, targets):
   packages = HostGccLibs(host).copy()
   for target in targets:
     packages.update(HostTools(host, target))
     if BuildTargetLibsOn(host):
       packages.update(TargetLibs(host, target))
-      packages.update(SDKLibs(host, target))
+      packages.update(SDKLibs(host, target, 'newlib'))
+      packages.update(SDKLibs(host, target, 'glibc'))
   return packages
 
 
 def CollectPackages(targets):
   packages = CollectSources()
 
   packages.update(CollectPackagesForHost(NATIVE_TUPLE, targets))
 
   for host in EXTRA_HOSTS_MAP.get(NATIVE_TUPLE, []):
     packages.update(CollectPackagesForHost(host, targets))
 
   return packages
 
 
 PACKAGES = CollectPackages(TARGET_LIST)
 PACKAGE_TARGETS = GetPackageTargets()
 
 
 if __name__ == '__main__':
   tb = toolchain_main.PackageBuilder(PACKAGES, PACKAGE_TARGETS, sys.argv[1:])
   # TODO(mcgrathr): The bot ought to run some native_client tests
   # using the new toolchain, like the old x86 toolchain bots do.
   tb.Main()