Switch PNaCl lib, include, and translator directories for new layout

toolchain_build/pnacl_targetlibs.py

Index: toolchain_build/pnacl_targetlibs.py
diff --git a/toolchain_build/pnacl_targetlibs.py b/toolchain_build/pnacl_targetlibs.py
index ed476f304e508de7ca17a0b86e9493f8dc87a84d..c3136b4fb5c456d3f0749f470da595f312e66915 100755
--- a/toolchain_build/pnacl_targetlibs.py
+++ b/toolchain_build/pnacl_targetlibs.py
@@ -20,6 +20,7 @@ import pnacl_commands
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
 NACL_DIR = os.path.dirname(SCRIPT_DIR)
 
+CLANG_VER = '3.4'
 
 # Return the path to the local copy of the driver script.
 # msys should be false if the path will be called directly rather than passed to
@@ -52,8 +53,8 @@ def MakeCommand():
 # Return the component name to use for a component name with
 # a host triple. GNU configuration triples contain dashes, which are converted
 # to underscores so the names are legal for Google Storage.
-def Mangle(component_name, extra):
-  return component_name + '_' + pynacl.gsd_storage.LegalizeName(extra)
+def GSDJoin(*args):
+  return '_'.join([pynacl.gsd_storage.LegalizeName(arg) for arg in args])
 
 
 # Copy the compiled bitcode archives used for linking C programs into the the
@@ -68,13 +69,16 @@ def CopyBitcodeCLibs(bias_arch):
       command.RemoveDirectory('usr'),
       command.Mkdir('usr'),
       command.Command('cp -r %(' +
-                      Mangle('abs_libs_support_bitcode', bias_arch) +
+                      GSDJoin('abs_libs_support_bitcode', bias_arch) +
                       ')s usr', shell=True),
-      command.Command('cp -r %(' + Mangle('abs_newlib', bias_arch) +
+      command.Command('cp -r %(' + GSDJoin('abs_newlib', bias_arch) +
                       ')s/* usr', shell=True),
       ]
 
 
+def BiasedBitcodeTriple(bias_arch):
+  return 'le32-nacl' if bias_arch == 'le32' else bias_arch + '_bc-nacl'
+
 def BiasedBitcodeTargetFlag(arch):
   flagmap = {
       # Arch     Target                           Extra flags.
@@ -89,14 +93,15 @@ def BiasedBitcodeTargetFlag(arch):
 
 def TargetBCLibCflags(bias_arch):
   flags = '-g -O2 -mllvm -inline-threshold=5'
-  if bias_arch != 'portable':
+  if bias_arch != 'le32':
     flags += ' ' + ' '.join(BiasedBitcodeTargetFlag(bias_arch))
   return flags
 
 def NewlibIsystemCflags(bias_arch):
   return ' '.join([
     '-isystem',
-    command.path.join('%(' + Mangle('abs_newlib', bias_arch) +')s', 'include')])
+    command.path.join('%(' + GSDJoin('abs_newlib', bias_arch) +')s',
+                      BiasedBitcodeTriple(bias_arch), 'include')])
 
 def LibCxxCflags(bias_arch):
   # HAS_THREAD_LOCAL is used by libc++abi's exception storage, the fallback is
@@ -113,13 +118,12 @@ def LibStdcxxCflags(bias_arch):
 # Build a single object file as bitcode.
 def BuildTargetBitcodeCmd(source, output, bias_arch, output_dir='%(cwd)s'):
   flags = ['-Wall', '-Werror', '-O2', '-c']
-  if bias_arch != 'portable':
-    [flags.append(flag) for flag in BiasedBitcodeTargetFlag(bias_arch)]
-  sysinclude = Mangle('newlib', bias_arch)
+  if bias_arch != 'le32':
+    flags.extend(BiasedBitcodeTargetFlag(bias_arch))
+  flags.extend(NewlibIsystemCflags(bias_arch).split())
   return command.Command(
-    [PnaclTool('clang', msys=False)] + flags + [
-        '-isystem', '%(' + sysinclude + ')s/include',
-     command.path.join('%(src)s', source),
+      [PnaclTool('clang', msys=False)] + flags + [
+          command.path.join('%(src)s', source),
      '-o', command.path.join(output_dir, output)])
 
 
@@ -132,7 +136,8 @@ def BuildTargetNativeCmd(sourcefile, output, arch, extra_flags=[],
      '-arch', arch, '--pnacl-bias=' + arch, '-O3',
      # TODO(dschuff): this include breaks the input encapsulation for build
      # rules.
-     '-I%(top_srcdir)s/..', '-isystem', '%(newlib_portable)s/include', '-c'] +
+     '-I%(top_srcdir)s/..','-c'] +
+    NewlibIsystemCflags('le32').split() +
     extra_flags +
     [command.path.join(source_dir, sourcefile),
      '-o', command.path.join(output_dir, output)])
@@ -234,16 +239,12 @@ def TargetLibsSrc(GitSyncCmds):
 
 def BitcodeLibs(bias_arch, is_canonical):
   def B(component_name):
-    return Mangle(component_name, bias_arch)
-  def BcSubdir(subdir, bias_arch):
-    if bias_arch == 'portable':
-      return subdir
-    else:
-      return subdir + '-bc-' + bias_arch
+    return GSDJoin(component_name, bias_arch)
+  bc_triple = BiasedBitcodeTriple(bias_arch)
+
   libs = {
       B('newlib'): {
           'type': 'build' if is_canonical else 'work',
-          'output_subdir': BcSubdir('usr', bias_arch),
           'dependencies': [ 'newlib_src', 'target_lib_compiler'],
           'commands' : [
               command.SkipForIncrementalCommand(
@@ -269,26 +270,28 @@ def BitcodeLibs(bias_arch, is_canonical):
               ]),
               command.Command(MakeCommand()),
               command.Command(['make', 'DESTDIR=%(abs_output)s', 'install']),
-              command.CopyTree(command.path.join('%(output)s', 'le32-nacl',
-                                                 'lib'),
-                               command.path.join('%(output)s', 'lib')),
-              command.CopyTree(
-                  command.path.join('%(output)s','le32-nacl', 'include'),
-                  command.path.join('%(output)s','include')),
-              command.RemoveDirectory(command.path.join('%(output)s',
-                                                        'le32-nacl')),
-              command.Mkdir(os.path.join('%(output)s', 'include', 'nacl')),
+              # We configured newlib with target=le32-nacl to get its pure C
+              # implementation, so rename its output dir (which matches the
+              # target to the output dir for the package we are building)
+              command.Rename(os.path.join('%(output)s', 'le32-nacl'),
+                             os.path.join('%(output)s', bc_triple)),
               # Copy nacl_random.h, used by libc++. It uses the IRT, so should
               # be safe to include in the toolchain.
+              command.Mkdir(
+                  os.path.join('%(output)s', bc_triple, 'include', 'nacl')),
               command.Copy(os.path.join('%(top_srcdir)s', 'src', 'untrusted',
                                         'nacl', 'nacl_random.h'),
-                           os.path.join('%(output)s', 'include', 'nacl',
-                                        'nacl_random.h')),
+                           os.path.join(
+                               '%(output)s', bc_triple, 'include', 'nacl',
+                               'nacl_random.h')),
+              # Remove the 'share' directory from the biased builds; the data is
+              # duplicated exactly and takes up 2MB per package.
+              command.RemoveDirectory(os.path.join('%(output)s', 'share'),
+                             run_cond = lambda x: bias_arch != 'le32'),
           ],
       },
       B('libcxx'): {
           'type': 'build' if is_canonical else 'work',
-          'output_subdir': BcSubdir('usr', bias_arch),
           'dependencies': ['libcxx_src', 'libcxxabi_src', 'llvm_src', 'gcc_src',
                            'target_lib_compiler', B('newlib'),
                            B('libs_support_bitcode')],
@@ -332,13 +335,15 @@ def BitcodeLibs(bias_arch, is_canonical):
                                         'unwind-generic.h'),
                            os.path.join('include', 'unwind.h')),
               command.Command(MakeCommand() + ['VERBOSE=1']),
-              command.Command(['make', 'DESTDIR=%(abs_output)s', 'VERBOSE=1',
-                               'install']),
+              command.Command([
+                  'make',
+                  'DESTDIR=' + os.path.join('%(abs_output)s', bc_triple),
+                  'VERBOSE=1',
+                  'install']),
           ],
       },
       B('libstdcxx'): {
           'type': 'build' if is_canonical else 'work',
-          'output_subdir': BcSubdir('usr', bias_arch),
           'dependencies': ['gcc_src', 'gcc_src', 'target_lib_compiler',
                            B('newlib')],
           'commands' : [
@@ -375,18 +380,23 @@ def BitcodeLibs(bias_arch, is_canonical):
                                         'unwind-generic.h'),
                            os.path.join('include', 'unwind.h')),
               command.Command(MakeCommand()),
-              command.Command(['make', 'DESTDIR=%(abs_output)s',
-                               'install-data']),
-              command.RemoveDirectory(os.path.join('%(output)s', 'share')),
-              command.Remove(os.path.join('%(output)s', 'lib',
+              command.Command([
+                  'make',
+                  'DESTDIR=' + os.path.join('%(abs_output)s', bc_triple),
+                  'install-data']),
+              command.RemoveDirectory(
+                  os.path.join('%(output)s', bc_triple, 'share')),
+              command.Remove(os.path.join('%(output)s', bc_triple, 'lib',
                                           'libstdc++*-gdb.py')),
-              command.Copy(os.path.join('src', '.libs', 'libstdc++.a'),
-                           os.path.join('%(output)s', 'lib', 'libstdc++.a')),
+              command.Copy(
+                  os.path.join('src', '.libs', 'libstdc++.a'),
+                  os.path.join('%(output)s', bc_triple, 'lib', 'libstdc++.a')),
           ],
       },
       B('libs_support_bitcode'): {
           'type': 'build' if is_canonical else 'work',
-          'output_subdir': BcSubdir('lib', bias_arch),
+          'output_subdir': os.path.join(
+              'lib', 'clang', CLANG_VER, 'lib', bc_triple),
           'dependencies': [ B('newlib'), 'target_lib_compiler'],
           'inputs': { 'src': os.path.join(NACL_DIR,
                                           'pnacl', 'support', 'bitcode')},
@@ -442,12 +452,13 @@ def NativeLibs(arch, is_canonical):
         [BuildTargetNativeCmd('entry_linux.c', 'entry_linux.o', arch),
          BuildTargetNativeCmd('entry_linux_arm.S', 'entry_linux_asm.o',
                               arch)])
+  native_lib_dir = os.path.join('translator', arch, 'lib')
 
   libs = {
-      Mangle('libs_support_native', arch): {
+      GSDJoin('libs_support_native', arch): {
           'type': 'build' if is_canonical else 'work',
-          'output_subdir': 'lib-' + arch,
-          'dependencies': [ 'newlib_src', 'newlib_portable',
+          'output_subdir': native_lib_dir,
+          'dependencies': [ 'newlib_src', 'newlib_le32',
                             'target_lib_compiler'],
           # These libs include
           # arbitrary stuff from native_client/src/{include,untrusted,trusted}
@@ -499,11 +510,11 @@ def NativeLibs(arch, is_canonical):
                                'dummy_shim_entry.o']),
           ],
       },
-      Mangle('compiler_rt', arch): {
+      GSDJoin('compiler_rt', arch): {
           'type': 'build' if is_canonical else 'work',
-          'output_subdir': 'lib-' + arch,
+          'output_subdir': native_lib_dir,
           'dependencies': ['compiler_rt_src', 'target_lib_compiler',
-                           'newlib_portable'],
+                           'newlib_le32'],
           'commands': [
               command.Command(MakeCommand() + [
                   '-f',
@@ -514,17 +525,17 @@ def NativeLibs(arch, is_canonical):
                   ['SRC_DIR=' + command.path.join('%(abs_compiler_rt_src)s',
                                                   'lib'),
                    'CFLAGS=-arch ' + arch + ' -DPNACL_' +
-                    arch.replace('-', '_') + ' --pnacl-allow-translate -O3' +
-                   ' -isystem %(newlib_portable)s/include']),
+                    arch.replace('-', '_') + ' --pnacl-allow-translate -O3 ' +
+                   NewlibIsystemCflags('le32')]),
               command.Copy('libgcc.a', os.path.join('%(output)s', 'libgcc.a')),
           ],
       },
   }
   if not arch.endswith('-nonsfi'):
     libs.update({
-      Mangle('libgcc_eh', arch): {
+      GSDJoin('libgcc_eh', arch): {
           'type': 'build' if is_canonical else 'work',
-          'output_subdir': 'lib-' + arch,
+          'output_subdir': native_lib_dir,
           'dependencies': [ 'gcc_src', 'target_lib_compiler'],
           'inputs': { 'scripts': os.path.join(NACL_DIR, 'pnacl', 'scripts')},
           'commands': [
@@ -551,9 +562,9 @@ def NativeLibs(arch, is_canonical):
 
 def UnsandboxedIRT(arch):
   libs = {
-      Mangle('unsandboxed_irt', arch): {
+      GSDJoin('unsandboxed_irt', arch): {
           'type': 'build',
-          'output_subdir': 'lib-' + arch,
+          'output_subdir': os.path.join('translator', arch, 'lib'),
           # This lib #includes
           # arbitrary stuff from native_client/src/{include,untrusted,trusted}
           'inputs': { 'support': os.path.join(NACL_DIR, 'src', 'nonsfi', 'irt'),