[NaCL SDK] Add initial support for nacl-clang

native_client_sdk/src/build_tools/build_sdk.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Entry point for both build and try bots.
 
 This script is invoked from XXX, usually without arguments
 to package an SDK. It automatically determines whether
 this SDK is for mac, win, linux.
@@ skipping 57 matching lines @@
     'newlib': ('nacl_x86_newlib', '%(platform)s_x86_newlib'),
     'bionic': ('nacl_arm_bionic', '%(platform)s_arm_bionic'),
     'arm': ('nacl_arm_newlib', '%(platform)s_arm_newlib'),
     'glibc': ('nacl_x86_glibc', '%(platform)s_x86_glibc'),
     'pnacl': ('pnacl_newlib', '%(platform)s_pnacl')
     }
 
 
 def GetToolchainNaClInclude(tcname, tcpath, arch):
   if arch == 'x86':
-    if tcname == 'pnacl':
-      return os.path.join(tcpath, 'le32-nacl', 'include')
     return os.path.join(tcpath, 'x86_64-nacl', 'include')
+  elif arch == 'pnacl':
+    return os.path.join(tcpath, 'le32-nacl', 'include')
   elif arch == 'arm':
     return os.path.join(tcpath, 'arm-nacl', 'include')
   else:
     buildbot_common.ErrorExit('Unknown architecture: %s' % arch)
 
 
 def GetConfigDir(arch):
-  if arch == 'x64' and getos.GetPlatform() == 'win':
+  if arch.endswith('x64') and getos.GetPlatform() == 'win':
     return 'Release_x64'
   else:
     return 'Release'
 
 
 def GetNinjaOutDir(arch):
   return os.path.join(OUT_DIR, GYPBUILD_DIR + '-' + arch, GetConfigDir(arch))
 
 
 def GetGypBuiltLib(tcname, arch):
   if arch == 'ia32':
     lib_suffix = '32'
   elif arch == 'x64':
     lib_suffix = '64'
   elif arch == 'arm':
     lib_suffix = 'arm'
   else:
     lib_suffix = ''
 
   if tcname == 'pnacl':
-    tcname = 'pnacl_newlib'
-    arch = 'x64'
+    print arch

[binji, 2015/02/20 18:34:23]

remove?

+    if arch is None:
+      arch = 'x64'
+      tcname = 'pnacl_newlib'
+    else:
+      arch = 'clang-' + arch
+      tcname = 'newlib'
 
   return os.path.join(GetNinjaOutDir(arch),
                       'gen',
                       'tc_' + tcname,
                       'lib' + lib_suffix)
 
 
 def GetToolchainNaClLib(tcname, tcpath, arch):
-  if tcname == 'pnacl':
-    return os.path.join(tcpath, 'le32-nacl', 'lib')
-  elif arch == 'ia32':
+  if arch == 'ia32':
     return os.path.join(tcpath, 'x86_64-nacl', 'lib32')
   elif arch == 'x64':
     return os.path.join(tcpath, 'x86_64-nacl', 'lib')
   elif arch == 'arm':
     return os.path.join(tcpath, 'arm-nacl', 'lib')
+  elif tcname == 'pnacl':
+    return os.path.join(tcpath, 'le32-nacl', 'lib')
 
 
 def GetToolchainDirName(tcname, arch):
   if tcname == 'pnacl':
     return '%s_%s' % (getos.GetPlatform(), tcname)
   elif arch == 'arm':
     return '%s_arm_%s' % (getos.GetPlatform(), tcname)
   else:
     return '%s_x86_%s' % (getos.GetPlatform(), tcname)
 
@@ skipping 182 matching lines @@
       else:
         dest = dest_file
       dest = os.path.join(dest_root, dest)
       if not os.path.isdir(os.path.dirname(dest)):
         buildbot_common.MakeDir(os.path.dirname(dest))
       buildbot_common.CopyFile(source, dest)
 
 
 def InstallNaClHeaders(tc_dst_inc, tc_name):
   """Copies NaCl headers to expected locations in the toolchain."""
-  if tc_name == 'arm':
-    # arm toolchain header should be the same as the x86 newlib
+  if tc_name in ('arm', 'pnacl'):
+    # arm and pnacl toolchain headers should be the same as the newlib
     # ones
     tc_name = 'newlib'
 
   InstallFiles(SRC_DIR, tc_dst_inc, NACL_HEADER_MAP[tc_name])
 
 
 def MakeNinjaRelPath(path):
   return os.path.join(os.path.relpath(OUT_DIR, SRC_DIR), path)
 
 
@@ skipping 95 matching lines @@
       ['irt_core_newlib_arm.nexe', 'irt_core_arm.nexe'],
       ['irt_core_newlib_arm.nexe', 'irt_core_arm.nexe'],
       ['nacl_helper_bootstrap', 'nacl_helper_bootstrap_arm'],
       ['nonsfi_loader_newlib_arm_nonsfi.nexe', 'nonsfi_loader_arm'],
       ['sel_ldr', 'sel_ldr_arm']
     ]
     InstallFiles(GetNinjaOutDir('arm'), tools_dir, arm_files)
 
   for tc in set(toolchains) & set(['newlib', 'glibc', 'pnacl']):
     if tc == 'pnacl':
-      xarches = (None,)
+      xarches = (None, 'ia32', 'x64')

[bradn, 2015/02/17 20:21:48]

why isn't arm in here?
I thought it now works for arm?

[Sam Clegg, 2015/02/17 20:31:12]

See my other comment.

+    elif tc == 'glibc':
+      xarches = ('ia32', 'x64')
     else:
       xarches = ('arm', 'ia32', 'x64')
 
     for xarch in xarches:
-      if tc == 'glibc' and xarch == 'arm':
-        continue
-
       src_dir = GetGypBuiltLib(tc, xarch)
       dst_dir = GetOutputToolchainLib(pepperdir, tc, xarch)
       InstallFiles(src_dir, dst_dir, TOOLCHAIN_LIBS[tc])
 
       # Copy ARM newlib components to bionic
       if tc == 'newlib' and xarch == 'arm' and 'bionic' in toolchains:
         bionic_dir = GetOutputToolchainLib(pepperdir, 'bionic', xarch)
         InstallFiles(src_dir, bionic_dir, TOOLCHAIN_LIBS['bionic'])
 
       if tc != 'pnacl':
@@ skipping 10 matching lines @@
   #
   # For now, let's use gyp_chromium to build these components.
 #  gyp_py = os.path.join(NACL_DIR, 'build', 'gyp_nacl')
   gyp_py = os.path.join(SRC_DIR, 'build', 'gyp_chromium')
   nacl_core_sdk_gyp = os.path.join(NACL_DIR, 'build', 'nacl_core_sdk.gyp')
   all_gyp = os.path.join(NACL_DIR, 'build', 'all.gyp')
 
   out_dir_32 = MakeNinjaRelPath(rel_out_dir + '-ia32')
   out_dir_64 = MakeNinjaRelPath(rel_out_dir + '-x64')
   out_dir_arm = MakeNinjaRelPath(rel_out_dir + '-arm')
+  out_dir_clang_32 = MakeNinjaRelPath(rel_out_dir + '-clang-ia32')
+  out_dir_clang_64 = MakeNinjaRelPath(rel_out_dir + '-clang-x64')
+
   GypNinjaBuild('ia32', gyp_py, nacl_core_sdk_gyp, 'nacl_core_sdk', out_dir_32)
+  GypNinjaBuild('x64', gyp_py, nacl_core_sdk_gyp, 'nacl_core_sdk', out_dir_64)
+  GypNinjaBuild('ia32', gyp_py, nacl_core_sdk_gyp, 'nacl_core_sdk',
+      out_dir_clang_32, gyp_defines=['use_nacl_clang=1'])
+  GypNinjaBuild('x64', gyp_py, nacl_core_sdk_gyp, 'nacl_core_sdk',
+      out_dir_clang_64, gyp_defines=['use_nacl_clang=1'])
   GypNinjaBuild('arm', gyp_py, nacl_core_sdk_gyp, 'nacl_core_sdk', out_dir_arm)
-  GypNinjaBuild('x64', gyp_py, nacl_core_sdk_gyp, 'nacl_core_sdk', out_dir_64)
   GypNinjaBuild('x64', gyp_py, all_gyp, 'ncval_new', out_dir_64)
 
 
 def GypNinjaBuild_Breakpad(rel_out_dir):
   # TODO(binji): dump_syms doesn't currently build on Windows. See
   # http://crbug.com/245456
   if getos.GetPlatform() == 'win':
     return
 
   gyp_py = os.path.join(SRC_DIR, 'build', 'gyp_chromium')
   out_dir = MakeNinjaRelPath(rel_out_dir)
   gyp_file = os.path.join(SRC_DIR, 'breakpad', 'breakpad.gyp')
   build_list = ['dump_syms', 'minidump_dump', 'minidump_stackwalk']
   GypNinjaBuild('x64', gyp_py, gyp_file, build_list, out_dir)
 
 
-def GypNinjaBuild_PPAPI(arch, rel_out_dir):
+def GypNinjaBuild_PPAPI(arch, rel_out_dir, gyp_defines=None):
   gyp_py = os.path.join(SRC_DIR, 'build', 'gyp_chromium')
   out_dir = MakeNinjaRelPath(rel_out_dir)
   gyp_file = os.path.join(SRC_DIR, 'ppapi', 'native_client',
                           'native_client.gyp')
-  GypNinjaBuild(arch, gyp_py, gyp_file, 'ppapi_lib', out_dir)
+  GypNinjaBuild(arch, gyp_py, gyp_file, 'ppapi_lib', out_dir,
+                gyp_defines=gyp_defines)
 
 
 def GypNinjaBuild_Pnacl(rel_out_dir, target_arch):
   # TODO(binji): This will build the pnacl_irt_shim twice; once as part of the
   # Chromium build, and once here. When we move more of the SDK build process
   # to gyp, we can remove this.
   gyp_py = os.path.join(SRC_DIR, 'build', 'gyp_chromium')
 
   out_dir = MakeNinjaRelPath(rel_out_dir)
   gyp_file = os.path.join(SRC_DIR, 'ppapi', 'native_client', 'src',
                           'untrusted', 'pnacl_irt_shim', 'pnacl_irt_shim.gyp')
   targets = ['aot']
   GypNinjaBuild(target_arch, gyp_py, gyp_file, targets, out_dir)
 
 
-def GypNinjaBuild(arch, gyp_py_script, gyp_file, targets, out_dir):
+def GypNinjaBuild(arch, gyp_py_script, gyp_file, targets,
+                  out_dir, force_arm_gcc=True, gyp_defines=None):

[binji, 2015/02/20 18:34:23]

force_arm_gcc is never used...?

   gyp_env = dict(os.environ)
   gyp_env['GYP_GENERATORS'] = 'ninja'
-  gyp_defines = ['nacl_allow_thin_archives=0']
+  gyp_defines = gyp_defines or []
+  gyp_defines.append('nacl_allow_thin_archives=0')
   if options.mac_sdk:
     gyp_defines.append('mac_sdk=%s' % options.mac_sdk)
+
   if arch is not None:
     gyp_defines.append('target_arch=%s' % arch)
     if arch == 'arm':
       gyp_env['GYP_CROSSCOMPILE'] = '1'
-      gyp_defines += ['arm_float_abi=hard']
+      gyp_defines.append('arm_float_abi=hard')
       if options.no_arm_trusted:
         gyp_defines.append('disable_cross_trusted=1')
   if getos.GetPlatform() == 'mac':
     gyp_defines.append('clang=1')
 
   gyp_env['GYP_DEFINES'] = ' '.join(gyp_defines)
   # We can't use windows path separators in GYP_GENERATOR_FLAGS since
   # gyp uses shlex to parse them and treats '\' as an escape char.
   gyp_env['GYP_GENERATOR_FLAGS'] = 'output_dir=%s' % out_dir.replace('\\', '/')
 
   # Print relevant environment variables
   for key, value in gyp_env.iteritems():
     if key.startswith('GYP') or key in ('CC',):
-      print '%s="%s"' % (key, value)
+      print '  %s="%s"' % (key, value)
 
   buildbot_common.Run(
       [sys.executable, gyp_py_script, gyp_file, '--depth=.'],
       cwd=SRC_DIR,
       env=gyp_env)
 
   NinjaBuild(targets, out_dir, arch)
 
 
 def NinjaBuild(targets, out_dir, arch):
@@ skipping 15 matching lines @@
     GypNinjaBuild_Breakpad(GYPBUILD_DIR + '-x64')
 
     if set(toolchains) & set(['glibc', 'newlib']):
       GypNinjaBuild_PPAPI('ia32', GYPBUILD_DIR + '-ia32')
       GypNinjaBuild_PPAPI('x64', GYPBUILD_DIR + '-x64')
 
     if 'arm' in toolchains:
       GypNinjaBuild_PPAPI('arm', GYPBUILD_DIR + '-arm')
 
     if 'pnacl' in toolchains:
+      GypNinjaBuild_PPAPI('ia32', GYPBUILD_DIR + '-clang-ia32',
+                          ['use_nacl_clang=1'])
+      GypNinjaBuild_PPAPI('x64', GYPBUILD_DIR + '-clang-x64',
+                          ['use_nacl_clang=1'])

[bradn, 2015/02/17 20:21:47]

arm?

[Sam Clegg, 2015/02/17 20:31:12]

When I started this change there was no arm support.  I could add it now if you
think thats is better, or add it in another CL.

Trying it locally now.

+
       # NOTE: For ia32, gyp builds both x86-32 and x86-64 by default.
       for arch in ('ia32', 'arm'):
         # Fill in the latest native pnacl shim library from the chrome build.
         build_dir = GYPBUILD_DIR + '-pnacl-' + arch
         GypNinjaBuild_Pnacl(build_dir, arch)
 
   GypNinjaInstall(pepperdir, toolchains)
 
   platform = getos.GetPlatform()
   newlibdir = os.path.join(pepperdir, 'toolchain', platform + '_x86_newlib')
   glibcdir = os.path.join(pepperdir, 'toolchain', platform + '_x86_glibc')
   armdir = os.path.join(pepperdir, 'toolchain', platform + '_arm_newlib')
   pnacldir = os.path.join(pepperdir, 'toolchain', platform + '_pnacl')
   bionicdir = os.path.join(pepperdir, 'toolchain', platform + '_arm_bionic')
 
+
   if 'newlib' in toolchains:
     InstallNaClHeaders(GetToolchainNaClInclude('newlib', newlibdir, 'x86'),
                        'newlib')
 
   if 'glibc' in toolchains:
     InstallNaClHeaders(GetToolchainNaClInclude('glibc', glibcdir, 'x86'),
                        'glibc')
 
   if 'arm' in toolchains:
     InstallNaClHeaders(GetToolchainNaClInclude('newlib', armdir, 'arm'),
@@ skipping 21 matching lines @@
 
         pnacl_translator_lib_dir = GetPNaClTranslatorLib(pnacldir, nacl_arch)
         if not os.path.isdir(pnacl_translator_lib_dir):
           buildbot_common.ErrorExit('Expected %s directory to exist.' %
                                     pnacl_translator_lib_dir)
 
         buildbot_common.CopyFile(
             os.path.join(release_build_dir, 'libpnacl_irt_shim.a'),
             pnacl_translator_lib_dir)
 
+    InstallNaClHeaders(GetToolchainNaClInclude('pnacl', pnacldir, 'pnacl'),
+                       'pnacl')

[bradn, 2015/02/17 20:21:48]

Why do these both have pnacl on the end?

[Sam Clegg, 2015/02/17 20:31:12]

See NACL_HEADER_MAP.

In both cases the set of headers we are installing are the pnacl
headers, which are basically the newlib headers.

We install them in le32-nacl/include and in x86_64-nacl/include
but they are the same headers.

     InstallNaClHeaders(GetToolchainNaClInclude('pnacl', pnacldir, 'x86'),
-                       'newlib')
+                       'pnacl')
 
 
 def MakeDirectoryOrClobber(pepperdir, dirname, clobber):
   dirpath = os.path.join(pepperdir, dirname)
   if clobber:
     buildbot_common.RemoveDir(dirpath)
   buildbot_common.MakeDir(dirpath)
 
   return dirpath
 
@@ skipping 73 matching lines @@
 
   if not os.path.isabs(output_filename):
     output_filename = os.path.join(fileroot, output_filename)
   generate_notice.Generate(output_filename, fileroot, license_files)
 
 
 def BuildStepVerifyFilelist(pepperdir):
   buildbot_common.BuildStep('Verify SDK Files')
   file_list_path = os.path.join(SCRIPT_DIR, 'sdk_files.list')
   try:
+    print 'SDK directory: %s' % pepperdir
     verify_filelist.Verify(file_list_path, pepperdir)
     print 'OK'
   except verify_filelist.ParseException, e:
     buildbot_common.ErrorExit('Parsing sdk_files.list failed:\n\n%s' % e)
   except verify_filelist.VerifyException, e:
     file_list_rel = os.path.relpath(file_list_path)
     verify_filelist_py = os.path.splitext(verify_filelist.__file__)[0] + '.py'
     verify_filelist_py = os.path.relpath(verify_filelist_py)
     pepperdir_rel = os.path.relpath(pepperdir)
 
@@ skipping 232 matching lines @@
     options.archive = True
     options.build_ports = True
     # TODO(binji): re-enable app_engine build when the linux builder stops
     # breaking when trying to git clone from github.
     # See http://crbug.com/412969.
     options.build_app_engine = False
     options.tar = True
 
   # NOTE: order matters here. This will be the order that is specified in the
   # Makefiles; the first toolchain will be the default.
-  toolchains = ['pnacl', 'newlib', 'glibc', 'arm', 'host']
+  toolchains = ['pnacl', 'newlib', 'glibc', 'arm', 'clang-newlib', 'host']
 
   # Changes for experimental bionic builder
   if options.bionic:
     toolchains.append('bionic')
     options.build_ports = False
     options.build_app_engine = False
 
   print 'Building: ' + ' '.join(toolchains)
 
   if options.archive and not options.tar:
@@ skipping 81 matching lines @@
     BuildStepArchiveSDKTools()
 
   return 0
 
 
 if __name__ == '__main__':
   try:
     sys.exit(main(sys.argv[1:]))
   except KeyboardInterrupt:
     buildbot_common.ErrorExit('build_sdk: interrupted')