Further deduplicate and simplify PNaCl SCons testing

buildbot/buildbot_pnacl.py

Index: buildbot/buildbot_pnacl.py
diff --git a/buildbot/buildbot_pnacl.py b/buildbot/buildbot_pnacl.py
index ed585200264b5d85cabfffe710cbb459a88533c0..84e9e7969ddf0ebe0226e6e11926111de69595ca 100755
--- a/buildbot/buildbot_pnacl.py
+++ b/buildbot/buildbot_pnacl.py
@@ -3,21 +3,16 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-import os
-
 from buildbot_lib import (
     BuildContext, BuildStatus, ParseStandardCommandLine,
-    RemoveSconsBuildDirectories, RemoveGypBuildDirectories, RunBuild,
-    SetupLinuxEnvironment, SetupMacEnvironment, SetupWindowsEnvironment, SCons,
-    Step )
-
+    RemoveSconsBuildDirectories, RunBuild, SetupLinuxEnvironment,
+    SetupMacEnvironment, SetupWindowsEnvironment, SCons, Step )
 
 
-def BuildScriptX86(status, context):
+def RunSconsTests(status, context):
   # Clean out build directories.
-  with Step('clobber', status):
+  with Step('clobber scons', status):
     RemoveSconsBuildDirectories()
-    RemoveGypBuildDirectories()

[Mark Seaborn, 2014/07/28 21:12:31]

What's the reason for removing this?  It doesn't seem to be moved elsewhere.

[Derek Schuff, 2014/07/28 21:16:05]

There's no gyp build anywhere in this script.

 
   # Unlike their arm counterparts we do not run trusted tests on x86 bots.
   # Trusted tests get plenty of coverage by other bots, e.g. nacl-gcc bots.
@@ -61,75 +56,87 @@ def BuildScriptX86(status, context):
     # The mac standalone sandboxed translator is flaky.
     # https://code.google.com/p/nativeclient/issues/detail?id=3856
 
-    with Step('toolchain_tests_sandboxed_translator', status,
+    if context['default_scons_platform'] == 'arm':
+      # The ARM sandboxed translator is flaky under qemu.
+      sbtc_tests = 'run_hello_world_test'
+    else:
+      sbtc_tests = 'toolchain_tests'
+
+    with Step(sbtc_tests + '_sandboxed_translator', status,
               halt_on_fail=False):
       SCons(context, parallel=True, mode=irt_mode,
             args=flags_run + ['use_sandboxed_translator=1',
-                              'toolchain_tests_irt'])
-    with Step('toolchain_tests_sandboxed_fast', status, halt_on_fail=False):
+                              sbtc_tests + '_irt'])
+    with Step(sbtc_tests + '_sandboxed_fast', status, halt_on_fail=False):
       SCons(context, parallel=True, mode=irt_mode,
             args=flags_run + ['use_sandboxed_translator=1', 'translate_fast=1',
-                              'toolchain_tests_irt'])
+                              sbtc_tests + '_irt'])
 
     # Translator memory consumption regression test
     with Step('large_code_test', status, halt_on_fail=False):
       SCons(context, parallel=True, mode=irt_mode,
             args=flags_run + ['use_sandboxed_translator=1', 'large_code'])
 
-  # Test Non-SFI Mode.
-  # The only architectures that the PNaCl toolchain supports Non-SFI
-  # versions of are currently x86-32 and ARM, and ARM testing is covered
-  # by buildbot_pnacl.sh rather than this Python script.
-  # The x86-64 toolchain bot currently also runs these tests from
-  # buildbot_pnacl.sh
-  if context.Linux() and context['default_scons_platform'] == 'x86-32':
-    with Step('nonsfi_tests', status, halt_on_fail=False):
-      # TODO(mseaborn): Enable more tests here when they pass.
-      tests = ['run_' + test + '_test_irt' for test in
-               ['float',
-                'hello_world',
-                'irt_futex',
-                'malloc_realloc_calloc_free',
-                'stack_alignment',
-                'syscall',
-                'thread']]
-      # Extra non-IRT-using test to run for x86-32
-      tests.extend(['run_clock_get_test',
-                    'run_dup_test',
-                    'run_fcntl_test',
-                    'run_fork_test',
-                    'run_hello_world_test',
-                    'run_mmap_test',
-                    'run_nanosleep_test',
-                    'run_prctl_test',
-                    'run_printf_test',
-                    'run_pwrite_test',
-                    'run_socket_test',
-                    'run_stack_alignment_test',
-                    'run_syscall_test',
-                    'run_thread_test'])
-      SCons(context, parallel=True, mode=irt_mode,
-            args=flags_run + ['nonsfi_nacl=1'] + tests)
+  if context.Linux() and (context['default_scons_platform'] == 'x86-32' or
+                          context['default_scons_platform'] == 'arm'):
+    # Test Non-SFI Mode.
+    # The only architectures that the PNaCl toolchain supports Non-SFI
+    # versions of are currently x86-32 and ARM, and ARM testing is covered
+    # by buildbot_pnacl.sh rather than this Python script.
+    # The x86-64 toolchain bot also runs these tests from buildbot_pnacl.sh
+
+    # TODO(mseaborn): Run small_tests_irt with nonsfi_nacl=1 when it passes,
+    # instead of the following whitelists of tests.
+    # These tests pass with both host libc and newlib, on x86-32 and ARM.
+    nonsfi_tests_common  = ['run_' + test + '_test_irt' for test in
+                            'dup',
+                            'float',
+                            'hello_world'
+                            'malloc_realloc_calloc_free',
+                            'stack_alignment',
+                            'syscall']
+    # These tests pass with the host libc, on x86-32 and ARM.
+    nonsfi_tests_host_libc = (nonsfi_tests_common +
+                              ['run_getpid_test_irt', 'toolchain_tests_irt'] +
+                              ['run_' + test + 'test' for test in
+                               'clock_get',
+                               'dup',
+                               'fcntl',
+                               'fork',
+                               'hello_world',
+                               'nanosleep',
+                               'prctl',
+                               'printf',
+                               'pwrite',
+                               'stack_alignment',
+                               'syscall'])
+    # These tests pass (with host libc) on x86-32 but not ARM
+    nonsfi_tests_host_libc_x8632 = (nonsfi_tests_host_libc +
+                                    ['run_mmap_test', 'run_socket_test'])
+    if context['default_scons_platform'] == 'x86-32':
+      # This uses the newlib-based nonsfi_loader, for which only a subset of
+      # the tests works so far.
+      # TODO(hamaji): Enable more tests.
+      with Step('nonsfi_tests_newlib', status, halt_on_fail=False):
+        SCons(context, parallel=True, mode=irt_mode,
+              args=flags_run + ['nonsfi_nacl=1'] + nonsfi_tests_common)
+
+      # Use x86-32 host libc tests below.
+      host_libc_tests = nonsfi_tests_host_libc_x8632
+    else:
+      # Use ARM host libc tests below.
+      host_libc_tests = nonsfi_tests_host_libc
 
     # Test nonsfi_loader linked against host's libc.
     with Step('nonsfi_tests_host_libc', status, halt_on_fail=False):
-      tests = ['run_' + test + '_test_irt' for test in
-               ['dup',
-                'float',
-                'getpid',
-                'hello_world',
-                'irt_futex',
-                'malloc_realloc_calloc_free',
-                'syscall',
-                'thread']]
       # Using skip_nonstable_bitcode=1 here disables the tests for
       # zero-cost C++ exception handling, which don't pass for Non-SFI
       # mode yet because we don't build libgcc_eh for Non-SFI mode.
-      tests.extend(['toolchain_tests_irt',
-                    'skip_nonstable_bitcode=1'])
       SCons(context, parallel=True, mode=irt_mode,
-            args=(flags_run + ['nonsfi_nacl=1', 'use_newlib_nonsfi_loader=0'] +
-                  tests))
+            args=(flags_run + ['nonsfi_nacl=1',
+                               'skip_nonstable_bitcode=1',
+                               'use_newlib_nonsfi_loader=0'] +
+                  host_libc_tests))
 
   # Test unsandboxed mode.
   if ((context.Linux() or context.Mac()) and
@@ -162,7 +169,7 @@ def Main():
   else:
     raise Exception('Unsupported platform')
 
-  RunBuild(BuildScriptX86, status)
+  RunBuild(RunSconsTests, status)
 
 if __name__ == '__main__':
   Main()