[NaCl SDK] Add arm gcc toolchain support to the SDK

native_client_sdk/src/build_tools/build_sdk.py

Index: native_client_sdk/src/build_tools/build_sdk.py
diff --git a/native_client_sdk/src/build_tools/build_sdk.py b/native_client_sdk/src/build_tools/build_sdk.py
index 5a88109f4557ed7169e5d8f96921ad6a1b3987be..fcfe9da588864b9a211f94ef20a5bdb8007d2144 100755
--- a/native_client_sdk/src/build_tools/build_sdk.py
+++ b/native_client_sdk/src/build_tools/build_sdk.py
@@ -46,7 +46,6 @@ NACL_DIR = os.path.join(SRC_DIR, 'native_client')
 OUT_DIR = os.path.join(SRC_DIR, 'out')
 PPAPI_DIR = os.path.join(SRC_DIR, 'ppapi')
 
-
 # Add SDK make tools scripts to the python path.
 sys.path.append(os.path.join(SDK_SRC_DIR, 'tools'))
 sys.path.append(os.path.join(NACL_DIR, 'build'))
@@ -59,7 +58,6 @@ GSTORE = 'https://commondatastorage.googleapis.com/nativeclient-mirror/nacl/'
 MAKE = 'nacl_sdk/make_3_81/make.exe'
 CYGTAR = os.path.join(NACL_DIR, 'build', 'cygtar.py')
 
-
 options = None
 
 
@@ -98,8 +96,10 @@ def GetToolchainNaClInclude(tcname, tcpath, arch, xarch=None):
     if tcname == 'pnacl':
       return os.path.join(tcpath, 'newlib', 'sdk', 'include')
     return os.path.join(tcpath, 'x86_64-nacl', 'include')
+  elif arch == 'arm':
+    return os.path.join(tcpath, 'arm-nacl', 'include')
   else:
-    buildbot_common.ErrorExit('Unknown architecture.')
+    buildbot_common.ErrorExit('Unknown architecture: %s' % arch)
 
 
 def GetToolchainNaClLib(tcname, tcpath, arch, xarch):
@@ -110,7 +110,9 @@ def GetToolchainNaClLib(tcname, tcpath, arch, xarch):
       return os.path.join(tcpath, 'x86_64-nacl', 'lib32')
     if str(xarch) == '64':
       return os.path.join(tcpath, 'x86_64-nacl', 'lib')
-  buildbot_common.ErrorExit('Unknown architecture.')
+    if str(xarch) == 'arm':
+      return os.path.join(tcpath, 'arm-nacl', 'lib')
+  buildbot_common.ErrorExit('Unknown architecture: %s' % arch)
 
 
 def GetPNaClNativeLib(tcpath, arch):
@@ -140,9 +142,11 @@ def GetBuildArgs(tcname, tcpath, outdir, arch, xarch=None):
 
 
 def BuildStepDownloadToolchains(platform):
-  buildbot_common.BuildStep('Rerun hooks to get toolchains')
-  buildbot_common.Run(['gclient', 'runhooks'],
-                      cwd=SRC_DIR, shell=(platform == 'win'))
+  buildbot_common.BuildStep('Running download_toolchains.py')
+  download_script = os.path.join('build', 'download_toolchains.py')
+  buildbot_common.Run([sys.executable, download_script,
+                      '--no-arm-trusted', '--arm-untrusted', '--keep'],
+                      cwd=NACL_DIR)
 
 
 def BuildStepCleanPepperDirs(pepperdir, pepperdir_old):
@@ -194,6 +198,13 @@ def BuildStepUntarToolchains(pepperdir, platform, arch, toolchains):
     newlibdir = os.path.join(pepperdir, 'toolchain', tcname + '_newlib')
     buildbot_common.Move(srcdir, newlibdir)
 
+  if 'arm' in toolchains:
+    # Copy the existing arm toolchain from native_client tree
+    arm_toolchain = os.path.join(NACL_DIR, 'toolchain', 'linux_arm_newlib')
+    arm_toolchain_sdk = os.path.join(pepperdir, 'toolchain',
+                                     os.path.basename(arm_toolchain))
+    buildbot_common.CopyDir(arm_toolchain, arm_toolchain_sdk)
+
   if 'glibc' in toolchains:
     # Untar the glibc toolchains
     tarfile = GetGlibcToolchain(platform, arch)
@@ -265,6 +276,10 @@ HEADER_MAP = {
 
 def InstallHeaders(tc_dst_inc, pepper_ver, tc_name):
   """Copies NaCl headers to expected locations in the toolchain."""
+  if tc_name == 'arm':
+    # arm toolchain header should be the same as the x86 newlib
+    # ones
+    tc_name = 'newlib'
   tc_map = HEADER_MAP[tc_name]
   for filename in tc_map:
     src = os.path.join(NACL_DIR, tc_map[filename])
@@ -347,11 +362,8 @@ def MakeNinjaRelPath(path):
   return os.path.join(os.path.relpath(OUT_DIR, SRC_DIR), path)
 
 
-def GypNinjaBuild_X86(pepperdir, platform, toolchains):
+def GypNinjaInstall(pepperdir, platform, toolchains):
   build_dir = 'gypbuild'
-  GypNinjaBuild_X86_Nacl(platform, build_dir)
-  GypNinjaBuild_X86_Chrome(build_dir)
-
   ninja_out_dir = os.path.join(OUT_DIR, build_dir, 'Release')
   # src_file, dst_file, is_host_exe?
   tools_files = [
@@ -383,13 +395,22 @@ def GypNinjaBuild_X86(pepperdir, platform, toolchains):
         os.path.join(pepperdir, 'tools', dst))
 
   for tc in set(toolchains) & set(['newlib', 'glibc']):
-    for bits in '32', '64':
+    for archname in ('arm', '32', '64'):
+      if tc == 'glibc' and archname == 'arm':
+        continue
       tc_dir = 'tc_' + tc
-      lib_dir = 'lib' + bits
+      lib_dir = 'lib' + archname
+      if archname == 'arm':
+        build_dir = 'gypbuild-arm'
+        tcdir = '%s_arm_%s' % (platform, tc)
+      else:
+        build_dir = 'gypbuild'
+        tcdir = '%s_x86_%s' % (platform, tc)
+
+      ninja_out_dir = os.path.join(OUT_DIR, build_dir, 'Release')
       src_dir = os.path.join(ninja_out_dir, 'gen', tc_dir, lib_dir)
-      tcpath = os.path.join(pepperdir, 'toolchain',
-                            '%s_x86_%s' % (platform, tc))
-      dst_dir = GetToolchainNaClLib(tc, tcpath, 'x86', bits)
+      tcpath = os.path.join(pepperdir, 'toolchain', tcdir)
+      dst_dir = GetToolchainNaClLib(tc, tcpath, 'x86', archname)
 
       buildbot_common.MakeDir(dst_dir)
       buildbot_common.CopyDir(os.path.join(src_dir, '*.a'), dst_dir)
@@ -399,21 +420,21 @@ def GypNinjaBuild_X86(pepperdir, platform, toolchains):
       if tc == 'glibc':
         buildbot_common.CopyDir(os.path.join(src_dir, '*.so'), dst_dir)
 
-      # TODO(binji): temporary hack; copy crt1.o from sdk toolchain directory.
-      lib_dir = os.path.join(ninja_out_dir, 'gen', 'sdk', 'toolchain',
-                     '%s_x86_%s' % (platform, tc), 'x86_64-nacl', 'lib')
-      if bits == '32':
-        lib_dir += '32'
+      ninja_tcpath = os.path.join(ninja_out_dir, 'gen', 'sdk', 'toolchain',
+                      tcdir)
+      lib_dir = GetToolchainNaClLib(tc, ninja_tcpath, 'x86', archname)
       buildbot_common.CopyFile(os.path.join(lib_dir, 'crt1.o'), dst_dir)
 
 
-def GypNinjaBuild_X86_Nacl(platform, rel_out_dir):
+def GypNinjaBuild_Nacl(platform, rel_out_dir):
   gyp_py = os.path.join(NACL_DIR, 'build', 'gyp_nacl')
   nacl_core_sdk_gyp = os.path.join(NACL_DIR, 'build', 'nacl_core_sdk.gyp')
   all_gyp = os.path.join(NACL_DIR, 'build', 'all.gyp')
 
   out_dir = MakeNinjaRelPath(rel_out_dir)
+  out_dir_arm = MakeNinjaRelPath(rel_out_dir + '-arm')
   GypNinjaBuild('ia32', gyp_py, nacl_core_sdk_gyp, 'nacl_core_sdk', out_dir)
+  GypNinjaBuild('arm', gyp_py, nacl_core_sdk_gyp, 'nacl_core_sdk', out_dir_arm)
   GypNinjaBuild('ia32', gyp_py, all_gyp, 'ncval_x86_32', out_dir)
   GypNinjaBuild(None, gyp_py, all_gyp, 'ncval_arm', out_dir)
 
@@ -421,7 +442,7 @@ def GypNinjaBuild_X86_Nacl(platform, rel_out_dir):
     NinjaBuild('sel_ldr64', out_dir)
     NinjaBuild('ncval_x86_64', out_dir)
   elif platform == 'linux':
-    out_dir_64 = MakeNinjaRelPath(rel_out_dir + '_64')
+    out_dir_64 = MakeNinjaRelPath(rel_out_dir + '-64')
     GypNinjaBuild('x64', gyp_py, nacl_core_sdk_gyp, 'sel_ldr', out_dir_64)
     GypNinjaBuild('x64', gyp_py, all_gyp, 'ncval_x86_64', out_dir_64)
 
@@ -439,17 +460,17 @@ def GypNinjaBuild_X86_Nacl(platform, rel_out_dir):
           os.path.join(SRC_DIR, out_dir, 'Release', dst))
 
 
-def GypNinjaBuild_X86_Chrome(rel_out_dir):
+def GypNinjaBuild_Chrome(arch, rel_out_dir):
   gyp_py = os.path.join(SRC_DIR, 'build', 'gyp_chromium')
 
   out_dir = MakeNinjaRelPath(rel_out_dir)
   gyp_file = os.path.join(SRC_DIR, 'ppapi', 'ppapi_untrusted.gyp')
   targets = ['ppapi_cpp_lib', 'ppapi_gles2_lib']
-  GypNinjaBuild('ia32', gyp_py, gyp_file, targets, out_dir)
+  GypNinjaBuild(arch, gyp_py, gyp_file, targets, out_dir)
 
   gyp_file = os.path.join(SRC_DIR, 'ppapi', 'native_client',
                           'native_client.gyp')
-  GypNinjaBuild('ia32', gyp_py, gyp_file, 'ppapi_lib', out_dir)
+  GypNinjaBuild(arch, gyp_py, gyp_file, 'ppapi_lib', out_dir)
 
 
 def GypNinjaBuild_Pnacl(rel_out_dir, target_arch):
@@ -462,10 +483,11 @@ def GypNinjaBuild_Pnacl(rel_out_dir, target_arch):
   gyp_file = os.path.join(SRC_DIR, 'ppapi', 'native_client', 'src',
                           'untrusted', 'pnacl_irt_shim', 'pnacl_irt_shim.gyp')
   targets = ['pnacl_irt_shim']
-  GypNinjaBuild(target_arch, gyp_py, gyp_file, targets, out_dir)
+  GypNinjaBuild(target_arch, gyp_py, gyp_file, targets, out_dir, False)
 
 
-def GypNinjaBuild(arch, gyp_py_script, gyp_file, targets, out_dir):
+def GypNinjaBuild(arch, gyp_py_script, gyp_file, targets,
+                  out_dir, force_arm_gcc=True):
   gyp_env = copy.copy(os.environ)
   gyp_env['GYP_GENERATORS'] = 'ninja'
   gyp_defines = []
@@ -473,8 +495,16 @@ def GypNinjaBuild(arch, gyp_py_script, gyp_file, targets, out_dir):
     gyp_defines.append('mac_sdk=%s' % options.mac_sdk)
   if arch:
     gyp_defines.append('target_arch=%s' % arch)
+    if arch == 'arm':
+      gyp_defines += ['armv7=1', 'arm_thumb=0', 'arm_neon=1']
+      if force_arm_gcc:
+        gyp_defines += ['build_newlib_arm_gcc=1']
+        gyp_defines += ['build_newlib_arm_pnacl=0']
 
   gyp_env['GYP_DEFINES'] = ' '.join(gyp_defines)
+  for key in ['GYP_GENERATORS', 'GYP_DEFINES']:
+    value = gyp_env[key]
+    print '%s="%s"' % (key, value)

[Sam Clegg, 2012/11/30 19:05:11]

I added more logging here, which I found useful but maybe
unneeded?

   gyp_generator_flags = ['-G', 'output_dir=%s' % (out_dir,)]
   gyp_depth = '--depth=.'
   buildbot_common.Run(
@@ -492,55 +522,66 @@ def NinjaBuild(targets, out_dir):
   buildbot_common.Run(['ninja', '-C', out_config_dir] + targets, cwd=SRC_DIR)
 
 
-def BuildStepBuildToolchains(pepperdir, platform, arch, pepper_ver,
-                             toolchains):
+def BuildStepBuildToolchains(pepperdir, platform, pepper_ver, toolchains):
   buildbot_common.BuildStep('SDK Items')
 
-  tcname = platform + '_' + arch
+  GypNinjaBuild_Nacl(platform, 'gypbuild')
+
+  tcname = platform + '_x86'
   newlibdir = os.path.join(pepperdir, 'toolchain', tcname + '_newlib')
   glibcdir = os.path.join(pepperdir, 'toolchain', tcname + '_glibc')
   pnacldir = os.path.join(pepperdir, 'toolchain', tcname + '_pnacl')
 
   # Run scons TC build steps
-  if arch == 'x86':
-    if set(toolchains) & set(['newlib', 'glibc']):
-      GypNinjaBuild_X86(pepperdir, platform, toolchains)
-
-    if 'newlib' in toolchains:
-      InstallHeaders(GetToolchainNaClInclude('newlib', newlibdir, 'x86'),
-                     pepper_ver,
-                     'newlib')
-
-    if 'glibc' in toolchains:
-      InstallHeaders(GetToolchainNaClInclude('glibc', glibcdir, 'x86'),
-                     pepper_ver,
-                     'glibc')
-
-    if 'pnacl' in toolchains:
-      shell = platform == 'win'
-      buildbot_common.Run(
-          GetBuildArgs('pnacl', pnacldir, pepperdir, 'x86', '32'),
-          cwd=NACL_DIR, shell=shell)
-      buildbot_common.Run(
-          GetBuildArgs('pnacl', pnacldir, pepperdir, 'x86', '64'),
-          cwd=NACL_DIR, shell=shell)
-
-      for arch in ('ia32', 'arm'):
-        # Fill in the latest native pnacl shim library from the chrome build.
-        GypNinjaBuild_Pnacl('gypbuild-' + arch, arch)
-        pnacl_libdir_map = { 'ia32': 'x86-64', 'arm': 'arm' }
-        release_build_dir = os.path.join(OUT_DIR, 'gypbuild-' + arch,
-                                         'Release', 'gen', 'tc_pnacl_translate',
-                                         'lib-' + pnacl_libdir_map[arch])
-        buildbot_common.CopyFile(
-            os.path.join(release_build_dir, 'libpnacl_irt_shim.a'),
-            GetPNaClNativeLib(pnacldir, pnacl_libdir_map[arch]))
-
-      InstallHeaders(GetToolchainNaClInclude('pnacl', pnacldir, 'x86'),
-                     pepper_ver,
-                     'newlib')
-  else:
-    buildbot_common.ErrorExit('Missing arch %s' % arch)
+  if set(toolchains) & set(['glibc', 'newlib']):
+    GypNinjaBuild_Chrome('ia32', 'gypbuild')
+
+  if 'arm' in toolchains:
+    GypNinjaBuild_Chrome('arm', 'gypbuild-arm')
+
+  GypNinjaInstall(pepperdir, platform, toolchains)
+
+  if 'newlib' in toolchains:
+    InstallHeaders(GetToolchainNaClInclude('newlib', newlibdir, 'x86'),
+                   pepper_ver,
+                   'newlib')
+
+  if 'glibc' in toolchains:
+    InstallHeaders(GetToolchainNaClInclude('glibc', glibcdir, 'x86'),
+                   pepper_ver,
+                   'glibc')
+
+  if 'arm' in toolchains:
+    tcname = platform + '_arm_newlib'
+    armdir = os.path.join(pepperdir, 'toolchain', tcname)
+    InstallHeaders(GetToolchainNaClInclude('newlib', armdir, 'arm'),
+                   pepper_ver, 'arm')
+
+  if 'pnacl' in toolchains:
+    shell = platform == 'win'
+    buildbot_common.Run(
+        GetBuildArgs('pnacl', pnacldir, pepperdir, 'x86', '32'),
+        cwd=NACL_DIR, shell=shell)
+    buildbot_common.Run(
+        GetBuildArgs('pnacl', pnacldir, pepperdir, 'x86', '64'),
+        cwd=NACL_DIR, shell=shell)
+
+    for arch in ('ia32', 'arm'):
+      # Fill in the latest native pnacl shim library from the chrome build.
+      build_dir = 'gypbuild-pnacl-' + arch
+      GypNinjaBuild_Pnacl(build_dir, arch)
+      pnacl_libdir_map = { 'ia32': 'x86-64', 'arm': 'arm' }
+      release_build_dir = os.path.join(OUT_DIR, build_dir, 'Release',
+                                       'gen', 'tc_pnacl_translate',
+                                       'lib-' + pnacl_libdir_map[arch])
+
+      buildbot_common.CopyFile(
+          os.path.join(release_build_dir, 'libpnacl_irt_shim.a'),
+          GetPNaClNativeLib(pnacldir, pnacl_libdir_map[arch]))
+
+    InstallHeaders(GetToolchainNaClInclude('pnacl', pnacldir, 'x86'),
+                   pepper_ver,
+                   'newlib')
 
 
 def BuildStepCopyBuildHelpers(pepperdir, platform):
@@ -627,7 +668,8 @@ def BuildStepCopyExamples(pepperdir, toolchains, build_experimental, clobber):
 
   args = ['--dstroot=%s' % pepperdir, '--master']
   for toolchain in toolchains:
-    args.append('--' + toolchain)
+    if toolchain != 'arm':
+      args.append('--' + toolchain)
 
   for example in EXAMPLE_LIST:
     dsc = os.path.join(SDK_EXAMPLE_DIR, example, 'example.dsc')
@@ -640,6 +682,7 @@ def BuildStepCopyExamples(pepperdir, toolchains, build_experimental, clobber):
   if build_experimental:
     args.append('--experimental')
 
+  print "Generting Makefiles: %s" % str(args)

[Sam Clegg, 2012/11/30 19:05:11]

Extra logging, maybe not needed?

This is another on of the cases where we are calling out
to a python library, but calling it with command line args
which seems a little strange.  I think we should either call
as a subprocess, or call into as a library.

[binji, 2012/12/03 19:52:44]

I half agree with this -- on the one hand, it is ugly building commandline
parameters to pass to a python script. On the other, it is a defined interface
and adding a new library function would mean defining another interface that
could potentially break in different ways than the commandline interface.

   if generate_make.main(args):
     buildbot_common.ErrorExit('Failed to build examples.')
 
@@ -821,7 +864,7 @@ def main(args):
   parser.add_option('--experimental',
       help='build experimental examples and libraries', action='store_true',
       dest='build_experimental')
-  parser.add_option('--skip-toolchain', help='Skip toolchain download/untar',
+  parser.add_option('--skip-toolchain', help='Skip toolchain untar',
       action='store_true')
   parser.add_option('--mac_sdk',
       help='Set the mac_sdk (e.g. 10.6) to use when building with ninja.',
@@ -844,7 +887,7 @@ def main(args):
   if buildbot_common.IsSDKTrybot():
     options.run_tests = True
 
-  toolchains = ['newlib', 'glibc', 'pnacl', 'host']
+  toolchains = ['newlib', 'glibc', 'arm', 'pnacl', 'host']
   print 'Building: ' + ' '.join(toolchains)
 
   if options.archive and options.skip_tar:
@@ -867,14 +910,15 @@ def main(args):
     # of the build.
     del os.environ['NACL_SDK_ROOT']
 
-  if not options.skip_toolchain:
-    BuildStepDownloadToolchains(platform)
   BuildStepCleanPepperDirs(pepperdir, pepperdir_old)
   BuildStepMakePepperDirs(pepperdir, ['include', 'toolchain', 'tools'])
-  BuildStepCopyTextFiles(pepperdir, pepper_ver, clnumber)
+
   if not options.skip_toolchain:
+    BuildStepDownloadToolchains(platform)
     BuildStepUntarToolchains(pepperdir, platform, arch, toolchains)
-  BuildStepBuildToolchains(pepperdir, platform, arch, pepper_ver, toolchains)
+
+  BuildStepCopyTextFiles(pepperdir, pepper_ver, clnumber)
+  BuildStepBuildToolchains(pepperdir, platform, pepper_ver, toolchains)
   InstallHeaders(os.path.join(pepperdir, 'include'), None, 'libs')
   BuildStepCopyBuildHelpers(pepperdir, platform)
   BuildStepCopyExamples(pepperdir, toolchains, options.build_experimental, True)