Add nacl-clang testing to SCons

SConstruct

 #! -*- python -*-
 # Copyright (c) 2012 The Native Client Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import atexit
 import json
 import os
 import platform
 import re
@@ skipping 243 matching lines @@
 # NOTE Command-line arguments are a SCons-ism that is separate from
 # command-line options.  Options are prefixed by "-" or "--" whereas arguments
 # are not.  The function SetBitFromOption can be used for options.
 # NOTE This function must be called before the bits are used
 # NOTE This function must be called after all modifications of ARGUMENTS have
 # been performed. See: ExpandArguments
 def SetUpArgumentBits(env):
   BitFromArgument(env, 'bitcode', default=False,
     desc='We are building bitcode')
 
+  BitFromArgument(env, 'nacl_clang', default=False,
+    desc='Use native nacl-clang compiler')
+
   BitFromArgument(env, 'translate_fast', default=False,
     desc='When using pnacl TC (bitcode=1) use accelerated translation step')
 
   BitFromArgument(env, 'built_elsewhere', default=False,
     desc='The programs have already been built by another system')
 
   BitFromArgument(env, 'skip_trusted_tests', default=False,
     desc='Only run untrusted tests - useful for translator testing'
       ' (also skips tests of the IRT itself')
 
@@ skipping 959 matching lines @@
 
 
 def GetToolchainDir(env, platform_build_dir=None, toolchain_name=None,
                     target_arch=None, is_pnacl=None, lib_name=None):
   if platform_build_dir is None:
     platform_build_dir = env.GetPlatformBuildTargetDir()
 
   if toolchain_name is None:
     # Fill in default arguments based on environment.
     if is_pnacl is None:
-      is_pnacl = env.Bit('bitcode')
+      # For the purposes of finding the toolchain dir, nacl_clang is PNaCl.
+      is_pnacl = env.Bit('bitcode') or env.Bit('nacl_clang')
       if lib_name is None:
         if is_pnacl or not env.Bit('nacl_glibc'):
           lib_name = 'newlib'
         else:
           lib_name = 'glibc'
 
     if target_arch is None:
       target_arch = pynacl.platform.GetArch(GetTargetPlatform())
 
     if is_pnacl:
@@ skipping 1647 matching lines @@
 # Do this before the site_scons/site_tools/naclsdk.py stuff to pass it along.
 pre_base_env.Append(
     PNACL_BCLDFLAGS = ARGUMENTS.get('pnacl_bcldflags', '').split(':'))
 
 
 # The nacl_env is used to build native_client modules
 # using a special tool chain which produces platform
 # independent binaries
 # NOTE: this loads stuff from: site_scons/site_tools/naclsdk.py
 nacl_env = MakeArchSpecificEnv()
+# See comment below about libc++ and libpthread in NONIRT_LIBS.
+needs_pthread = nacl_env.Bit('bitcode') or nacl_env.Bit('nacl_clang')
 nacl_env = nacl_env.Clone(
     tools = ['naclsdk'],
     NACL_BUILD_FAMILY = 'UNTRUSTED',
     BUILD_TYPE = 'nacl',
     BUILD_TYPE_DESCRIPTION = 'NaCl module build',
 
     ARFLAGS = 'rc',
 
     # ${SOURCE_ROOT} for #include <ppapi/...>
     CPPPATH = [
@@ skipping 33 matching lines @@
                        '__env__, RDirs, TARGET, SOURCE)} $)',
     RPATHLINKPREFIX = '-Wl,-rpath-link,',
     RPATHLINKSUFFIX = '',
 
     LIBS = [],
     LINKFLAGS = ['${RPATH_LINK_FLAGS}'],
 
     # These are settings for in-tree, non-browser tests to use.
     # They use libraries that circumvent the IRT-based implementations
     # in the public libraries.
-    # Note that pthread_private is part of NONIRT_LIBS for PNaCl because
-    # libc++ depends on it.
+    # Note that pthread_private is part of NONIRT_LIBS for clang because
+    # libc++ depends on libpthread. However we can't just add
+    # libpthread_private to the link line because those libs get added before
+    # the standard libs, so the references that come from libc++ itself will
+    # stil get satisfied from libpthread instead of libpthread_private (and

[jvoung (off chromium), 2014/10/09 17:35:01]

"still get"

[Derek Schuff, 2014/10/09 18:40:40]

Done.

+    # that code will crash because it requires the IRT). So put libc++ on the
+    # user link line before libpthread_private to ensure that its references

[jvoung (off chromium), 2014/10/09 17:35:01]

I guess this is the actual hack that was in pnacl-translate... so this part
could help remove the hack later (though it hard-codes libc++, so might affect
the libstdc++ pnacl EH test variants... which are going to be removed
eventually).

[Derek Schuff, 2014/10/09 18:40:39]

Yeah the idea was to allow this to remove the hack in pnacl-ld also... but I had
forgotten about the libstdc++ EH tests. maybe i'll just use this for nacl_clang
for now and look at pnacl itself later.

+    # to libpthread also get satisfied by libpthread_private.
     NONIRT_LIBS = (['nacl_sys_private'] +
-                   (['pthread_private'] if nacl_env.Bit('bitcode') else [])),
+                   (['c++','pthread_private'] if needs_pthread else [])),
     PTHREAD_LIBS = ['pthread_private'],
     DYNCODE_LIBS = ['nacl_dyncode_private'],
     EXCEPTION_LIBS = ['nacl_exception_private'],
     LIST_MAPPINGS_LIBS = ['nacl_list_mappings_private'],
     RANDOM_LIBS = ['nacl_random_private'],
     )
 
 def UsesAbiNote(env):
   """Return True if using a new-style GCC with .note.NaCl.ABI.* notes.
 This means there will always be an RODATA segment, even if just for the note."""
@@ skipping 177 matching lines @@
 
   # With pnacl's clang base/ code uses the "override" keyword.
   nacl_env.Append(CXXFLAGS=['-Wno-c++11-extensions'])
   # Allow extraneous semicolons.  (Until these are removed.)
   # http://code.google.com/p/nativeclient/issues/detail?id=2861
   nacl_env.Append(CCFLAGS=['-Wno-extra-semi'])
   # Allow unused private fields.  (Until these are removed.)
   # http://code.google.com/p/nativeclient/issues/detail?id=2861
   nacl_env.Append(CCFLAGS=['-Wno-unused-private-field'])
 
+if nacl_env.Bit('nacl_clang'):
+  nacl_env.Append(CCFLAGS=['-Wno-format-security'])

[jvoung (off chromium), 2014/10/09 17:35:01]

Why is Wno-format-security needed for nacl_clang? I don't see that used for any
other clang variant?

Could the untrusted code be fixed?

[Derek Schuff, 2014/10/09 18:40:39]

Hm, i had it in there during bootstrapping for some reason but now it seems to
be working without it.

+  # third_party/valgrind/nacl_valgrind.h uses asm instead of __asm__
+  nacl_env.Append(CCFLAGS=['-Wno-language-extension-token'])
+
 # We use a special environment for building the IRT image because it must
 # always use the newlib toolchain, regardless of --nacl_glibc.  We clone
 # it from nacl_env here, before too much other cruft has been added.
 # We do some more magic below to instantiate it the way we need it.
 nacl_irt_env = nacl_env.Clone(
     BUILD_TYPE = 'nacl_irt',
     BUILD_TYPE_DESCRIPTION = 'NaCl IRT build',
     NACL_BUILD_FAMILY = 'UNTRUSTED_IRT',
 )
 
 # Provide access to the IRT build environment from the default environment
 # which is needed when compiling custom IRT for testing purposes.
 nacl_env['NACL_IRT_ENV'] = nacl_irt_env
 
 # Since we don't build src/untrusted/pthread/nacl.scons in
 # nacl_irt_env, we must tell the IRT how to find the pthread.h header.
 nacl_irt_env.Append(CPPPATH='${MAIN_DIR}/src/untrusted/pthread')
 
 # Map certain flag bits to suffices on the build output.  This needs to
 # happen pretty early, because it affects any concretized directory names.
 target_variant_map = [
     ('bitcode', 'pnacl'),
     ('translate_fast', 'fast'),
     ('nacl_pic', 'pic'),
     ('use_sandboxed_translator', 'sbtc'),
     ('nacl_glibc', 'glibc'),
     ('pnacl_generate_pexe', 'pexe'),
     ('nonsfi_nacl', 'nonsfi'),
+    ('nacl_clang', 'clang'),
     ]
 for variant_bit, variant_suffix in target_variant_map:
   if nacl_env.Bit(variant_bit):
     nacl_env['TARGET_VARIANT'] += '-' + variant_suffix
 
 if nacl_env.Bit('bitcode'):
   nacl_env['TARGET_VARIANT'] += '-clang'
 
 nacl_env.Replace(TESTRUNNER_LIBS=['testrunner'])
 # TODO(mseaborn): Drop this once chrome side has inlined this.
@@ skipping 331 matching lines @@
                                    '--pnacl-allow-translate',
                                    '-arch', 'x86-32',
                                    '-Wt,-mattr=-sse'])
   elif nacl_irt_env.Bit('target_mips32'):
     # Disable the PNaCl IRT verifier since it will complain about
     # __executable_start symbol not being a valid external symbol.
     nacl_irt_env.Append(LINKFLAGS=['--pnacl-disable-abi-check'])
 
 # The IRT is C only, don't link with the C++ linker so that it doesn't
 # start depending on the C++ standard library and (in the case of
 # libc++) pthread.

[jvoung (off chromium), 2014/10/09 17:35:00]

This was another hack, I think, from the libc++ dependency on pthread that could
be re-investigated.

[Derek Schuff, 2014/10/09 18:40:40]

It looks like we still want it to avoid depending on libc++, no? even if we do
always link with pthread?

[jvoung (off chromium), 2014/10/09 19:05:01]

Okay, and now it doesn't apply to plain bitcode=1 either.

 nacl_irt_env.Replace(LINK=(nacl_irt_env['LINK'].
                            replace('pnacl-clang++', 'pnacl-clang')))
 
 if nacl_irt_env.Bit('bitcode'):
   nacl_irt_env.Append(LINKFLAGS=['--pnacl-allow-native'])
 
 # All IRT code must avoid direct use of the TLS ABI register, which
 # is reserved for user TLS.  Instead, ensure all TLS accesses use a
 # call to __nacl_read_tp, which the IRT code overrides to segregate
 # IRT-private TLS from user TLS. This only applies to mips now, on
@@ skipping 97 matching lines @@
     if env.Bit('bitcode'):
       implicit_libs += ['libnacl.a']
     else:
       implicit_libs += ['crt1.o',
                         'libnacl.a',
                         'crti.o',
                         'crtn.o']
       # TODO(mcgrathr): multilib nonsense defeats -B!  figure out a better way.
       if GetTargetPlatform() == 'x86-32':
         implicit_libs.append(os.path.join('32', 'crt1.o'))
+    # libc++ depends on libpthread, and because PPAPI applications always need
+    # threads anyway, nacl-clang just includes -lpthread unconditionally.
+    if ((env.Bit('nacl_clang') or env.Bit('bitcode')) and

[jvoung (off chromium), 2014/10/09 17:35:01]

Can use "needs_pthread" here too

[Derek Schuff, 2014/10/09 18:40:39]

Done, and renamed.

+        env['NACL_BUILD_FAMILY'] != 'UNTRUSTED_IRT'):
+      implicit_libs += ['libpthread.a']
 
   if implicit_libs != []:
     env['IMPLICIT_LIBS'] = [env.File(os.path.join('${LIB_DIR}', file))
                             for file in implicit_libs]
     # The -B<dir>/ flag is necessary to tell gcc to look for crt[1in].o there.
     env.Prepend(LINKFLAGS=['-B${LIB_DIR}/'])
 
 AddImplicitLibs(nacl_env)
 AddImplicitLibs(nacl_irt_env)
 
@@ skipping 358 matching lines @@
   nacl_env.ValidateSdk()
 
 if BROKEN_TEST_COUNT > 0:
   msg = "There are %d broken tests." % BROKEN_TEST_COUNT
   if GetOption('brief_comstr'):
     msg += " Add --verbose to the command line for more information."
   print msg
 
 # separate warnings from actual build output
 Banner('B U I L D - O U T P U T:')