Further deduplicate and simplify PNaCl SCons testing

buildbot/buildbot_pnacl.py

 #!/usr/bin/python
 # Copyright (c) 2013 The Native Client Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-import os
-
 from buildbot_lib import (
     BuildContext, BuildStatus, ParseStandardCommandLine,
-    RemoveSconsBuildDirectories, RemoveGypBuildDirectories, RunBuild,
-    SetupLinuxEnvironment, SetupMacEnvironment, SetupWindowsEnvironment, SCons,
-    Step )
+    RemoveSconsBuildDirectories, RunBuild, SetupLinuxEnvironment,
+    SetupMacEnvironment, SetupWindowsEnvironment, SCons, Step )
 
 
-
-def BuildScriptX86(status, context):
+def RunSconsTests(status, context):
   # Clean out build directories.
-  with Step('clobber', status):
+  with Step('clobber scons', status):
     RemoveSconsBuildDirectories()
-    RemoveGypBuildDirectories()
 
   # Unlike their arm counterparts we do not run trusted tests on x86 bots.
   # Trusted tests get plenty of coverage by other bots, e.g. nacl-gcc bots.
   # We make the assumption here that there are no "exotic tests" which
   # are trusted in nature but are somehow depedent on the untrusted TC.
   flags_build = ['skip_trusted_tests=1', 'do_not_run_tests=1']
   flags_run = ['skip_trusted_tests=1']
   smoke_tests = ['small_tests', 'medium_tests']
 
-  with Step('build_all', status):
+  arch = context['default_scons_platform']
+
+  with Step('build_all ' + arch, status):
     SCons(context, parallel=True, args=flags_build)
 
   # Normal pexe-mode tests
-  with Step('smoke_tests', status, halt_on_fail=False):
+  with Step('smoke_tests ' + arch, status, halt_on_fail=False):
     SCons(context, parallel=True, args=flags_run + smoke_tests)
   # Large tests cannot be run in parallel
-  with Step('large_tests', status, halt_on_fail=False):
+  with Step('large_tests ' + arch, status, halt_on_fail=False):
     SCons(context, parallel=False, args=flags_run + ['large_tests'])
 
   # non-pexe-mode tests. Build everything to make sure it all builds in nonpexe
   # mode, but just run the nonpexe_tests
-  with Step('build_nonpexe', status):
+  with Step('build_nonpexe ' + arch, status):
     SCons(context, parallel=True, args=flags_build + ['pnacl_generate_pexe=0'])
-  with Step('nonpexe_tests', status, halt_on_fail=False):
+  with Step('nonpexe_tests ' + arch, status, halt_on_fail=False):
     SCons(context, parallel=True,
           args=flags_run + ['pnacl_generate_pexe=0', 'nonpexe_tests'])
 
   irt_mode = context['default_scons_mode'] + ['nacl_irt_test']
   smoke_tests_irt = ['small_tests_irt', 'medium_tests_irt']
   # Run some tests with the IRT
-  with Step('smoke_tests_irt', status, halt_on_fail=False):
+  with Step('smoke_tests_irt ' + arch, status, halt_on_fail=False):
     SCons(context, parallel=True, mode=irt_mode,
           args=flags_run + smoke_tests_irt)
 
   # Test sandboxed translation
   if not context.Windows() and not context.Mac():
     # TODO(dschuff): The standalone sandboxed translator driver does not have
     # the batch script wrappers, so it can't run on Windows. Either add them to
     # the translator package or make SCons use the pnacl_newlib drivers except
     # on the ARM bots where we don't have the pnacl_newlib drivers.
     # The mac standalone sandboxed translator is flaky.
     # https://code.google.com/p/nativeclient/issues/detail?id=3856
 
-    with Step('toolchain_tests_sandboxed_translator', status,
+    if arch == 'arm':
+      # The ARM sandboxed translator is flaky under qemu, so run a very small
+      # set of tests there.
+      sbtc_tests = ['run_hello_world_test_irt']
+    else:
+      sbtc_tests = ['toolchain_tests_irt', 'large_code']
+
+    with Step('sandboxed_translator_tests ' + arch, status,
+              halt_on_fail=False):
+      SCons(context, parallel=True, mode=irt_mode,
+            args=flags_run + ['use_sandboxed_translator=1'] + sbtc_tests)
+    with Step('sandboxed_translator_fast_tests' + arch, status,

[Mark Seaborn, 2014/07/31 15:52:03]

Missing space

[Derek Schuff, 2014/07/31 16:18:09]

Done.

               halt_on_fail=False):
       SCons(context, parallel=True, mode=irt_mode,
             args=flags_run + ['use_sandboxed_translator=1',
-                              'toolchain_tests_irt'])
-    with Step('toolchain_tests_sandboxed_fast', status, halt_on_fail=False):
+                              'translate_fast=1'] + sbtc_tests)
+
+  if context.Linux() and (arch == 'x86-32' or arch == 'arm'):
+    # Test Non-SFI Mode.
+    # The only architectures that the PNaCl toolchain supports Non-SFI
+    # versions of are currently x86-32 and ARM, and ARM testing is covered

[Mark Seaborn, 2014/07/31 15:52:03]

Need to remove "ARM testing is covered buildbot_pnacl.sh rather than this Python
script".

[Derek Schuff, 2014/07/31 16:18:09]

Done.

+    # by buildbot_pnacl.sh rather than this Python script.
+    # The x86-64 toolchain bot also runs these tests from buildbot_pnacl.sh
+
+    # TODO(mseaborn): Run small_tests_irt with nonsfi_nacl=1 when it passes,
+    # instead of the following whitelists of tests.
+    # These tests pass with both host libc and newlib, on x86-32 and ARM.
+    nonsfi_tests_common  = ['run_' + test + '_test_irt' for test in

[Mark Seaborn, 2014/07/31 15:52:03]

I found the Non-SFI part of the refactoring really difficult to review.  It's
hard for me to compare the lists of tests.

I'm going to send you a cleanup change that makes this a lot simpler:
https://codereview.chromium.org/424763007/

[Derek Schuff, 2014/07/31 16:18:09]

looks good.

+                            'dup',
+                            'float',
+                            'hello_world',
+                            'irt_futex',
+                            'malloc_realloc_calloc_free',
+                            'mmap',
+                            'stack_alignment',
+                            'syscall']
+    # These tests pass with the host libc, on x86-32 and ARM.
+    nonsfi_tests_host_libc = (nonsfi_tests_common +
+                              ['run_getpid_test_irt', 'toolchain_tests_irt'] +
+                              ['run_' + test + '_test' for test in
+                               'clock_get',
+                               'dup',
+                               'fcntl',
+                               'fork',
+                               'hello_world',
+                               'nanosleep',
+                               'prctl',
+                               'printf',
+                               'pwrite',
+                               'stack_alignment',
+                               'syscall'])
+    # These tests pass (with host libc) on x86-32 but not ARM
+    nonsfi_tests_host_libc_x8632 = (nonsfi_tests_host_libc +
+                                    ['run_mmap_test', 'run_socket_test'])
+
+    # This uses the newlib-based nonsfi_loader, for which only a subset of
+    # the tests works so far.
+    # TODO(hamaji): Enable more tests.
+    with Step('nonsfi_tests_newlib ' + arch, status, halt_on_fail=False):
       SCons(context, parallel=True, mode=irt_mode,
-            args=flags_run + ['use_sandboxed_translator=1', 'translate_fast=1',
-                              'toolchain_tests_irt'])
+            args=flags_run + ['nonsfi_nacl=1'] + nonsfi_tests_common)
 
-    # Translator memory consumption regression test
-    with Step('large_code_test', status, halt_on_fail=False):
-      SCons(context, parallel=True, mode=irt_mode,
-            args=flags_run + ['use_sandboxed_translator=1', 'large_code'])
-
-  # Test Non-SFI Mode.
-  # The only architectures that the PNaCl toolchain supports Non-SFI
-  # versions of are currently x86-32 and ARM, and ARM testing is covered
-  # by buildbot_pnacl.sh rather than this Python script.
-  # The x86-64 toolchain bot currently also runs these tests from
-  # buildbot_pnacl.sh
-  if context.Linux() and context['default_scons_platform'] == 'x86-32':
-    with Step('nonsfi_tests', status, halt_on_fail=False):
-      # TODO(mseaborn): Enable more tests here when they pass.
-      tests = ['run_' + test + '_test_irt' for test in
-               ['float',
-                'hello_world',
-                'irt_futex',
-                'malloc_realloc_calloc_free',
-                'mmap',
-                'stack_alignment',
-                'syscall',
-                'thread']]
-      # Extra non-IRT-using test to run for x86-32
-      tests.extend(['run_clock_get_test',
-                    'run_dup_test',
-                    'run_fcntl_test',
-                    'run_fork_test',
-                    'run_hello_world_test',
-                    'run_mmap_test',
-                    'run_nanosleep_test',
-                    'run_prctl_test',
-                    'run_printf_test',
-                    'run_pwrite_test',
-                    'run_socket_test',
-                    'run_stack_alignment_test',
-                    'run_syscall_test',
-                    'run_thread_test'])
-      SCons(context, parallel=True, mode=irt_mode,
-            args=flags_run + ['nonsfi_nacl=1'] + tests)
-
+    if arch == 'x86-32':
+      # Use x86-32 host libc tests.
+      host_libc_tests = nonsfi_tests_host_libc_x8632
+    else:
+      # Use ARM host libc tests.
+      host_libc_tests = nonsfi_tests_host_libc
     # Test nonsfi_loader linked against host's libc.
-    with Step('nonsfi_tests_host_libc', status, halt_on_fail=False):
-      tests = ['run_' + test + '_test_irt' for test in
-               ['dup',
-                'float',
-                'getpid',
-                'hello_world',
-                'irt_futex',
-                'malloc_realloc_calloc_free',
-                'syscall',
-                'thread']]
+    with Step('nonsfi_tests_host_libc ' + arch, status, halt_on_fail=False):
       # Using skip_nonstable_bitcode=1 here disables the tests for
       # zero-cost C++ exception handling, which don't pass for Non-SFI
       # mode yet because we don't build libgcc_eh for Non-SFI mode.
-      tests.extend(['toolchain_tests_irt',
-                    'skip_nonstable_bitcode=1'])
       SCons(context, parallel=True, mode=irt_mode,
-            args=(flags_run + ['nonsfi_nacl=1', 'use_newlib_nonsfi_loader=0'] +
-                  tests))
+            args=(flags_run + ['nonsfi_nacl=1',
+                               'skip_nonstable_bitcode=1',
+                               'use_newlib_nonsfi_loader=0'] +
+                  host_libc_tests))
 
   # Test unsandboxed mode.
-  if ((context.Linux() or context.Mac()) and
-      context['default_scons_platform'] == 'x86-32'):
+  if (context.Linux() or context.Mac()) and arch == 'x86-32':
     if context.Linux():
       tests = ['run_' + test + '_test_irt' for test in
                ['hello_world', 'irt_futex', 'thread', 'float',
                 'malloc_realloc_calloc_free', 'dup', 'cond_timedwait',
                 'getpid']]
     else:
       # TODO(mseaborn): Use the same test list as on Linux when the threading
       # tests pass for Mac.
       tests = ['run_hello_world_test_irt']
-    with Step('unsandboxed_tests', status, halt_on_fail=False):
+    with Step('unsandboxed_tests ' + arch, status, halt_on_fail=False):
       SCons(context, parallel=True, mode=irt_mode,
             args=flags_run + ['pnacl_unsandboxed=1'] + tests)
 
 
 def Main():
   context = BuildContext()
   status = BuildStatus(context)
   ParseStandardCommandLine(context)
 
   if context.Linux():
     SetupLinuxEnvironment(context)
   elif context.Windows():
     SetupWindowsEnvironment(context)
   elif context.Mac():
     SetupMacEnvironment(context)
   else:
     raise Exception('Unsupported platform')
 
-  RunBuild(BuildScriptX86, status)
+  RunBuild(RunSconsTests, status)
 
 if __name__ == '__main__':
   Main()