Further deduplicate and simplify PNaCl SCons testing

buildbot/buildbot_pnacl.sh

 #!/bin/bash
 # Copyright (c) 2012 The Native Client Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 set -o xtrace
 set -o nounset
 set -o errexit
 
 # Tell build.sh and test.sh that we're a bot.
@@ skipping 63 matching lines @@
 readonly SCONS_NONSFI="nonsfi_nacl=1 ${SCONS_NONSFI_TESTS}"
 
 # subset of tests used on toolchain builders
 readonly SCONS_TC_TESTS="small_tests medium_tests"
 
 readonly SCONS_COMMON="./scons --verbose bitcode=1"
 readonly UP_DOWN_LOAD="buildbot/file_up_down_load.sh"
 # This script is used by toolchain bots (i.e. tc-xxx functions)
 readonly PNACL_BUILD="pnacl/build.sh"
 readonly DRIVER_TESTS="pnacl/driver/tests/driver_tests.py"
+readonly BUILDBOT_PNACL_PY="buildbot/buildbot_pnacl.py"
 
 
 tc-build-translator() {
   echo @@@BUILD_STEP compile_translator@@@
   ${PNACL_BUILD} translator-clean-all
   ${PNACL_BUILD} translator-all
 }
 
 tc-prune-translator-pexes() {
   echo @@@BUILD_STEP prune_translator_pexe@@@
@@ skipping 247 matching lines @@
 
   scons-stage-irt "arm" \
     "${qemuflags} use_sandboxed_translator=1 translate_in_build_step=0" \
     "toolchain_tests"
   scons-stage-irt "arm" \
     "${qemuflags} use_sandboxed_translator=1 translate_fast=1 \
        translate_in_build_step=0" \
     "toolchain_tests"
 
   # Test Non-SFI Mode.
   scons-stage-irt "arm" "${qemuflags}" "${SCONS_NONSFI_NEWLIB}"

[Mark Seaborn, 2014/07/29 17:11:33]

Can you convert this (mode-buildbot-arm*) too?

Otherwise the ARM trybot and buildbot will be diverging further by being defined
in different files, with different test lists.  There would be a bigger chance
that a change will pass on the trybots but fail on the main bots.

[Derek Schuff, 2014/07/29 18:41:50]

The ARM buildbot still matches the nacl-arm_opt_panda trybot, and uses this
code. The change in this cl only affects the
nacl-precise_64-newlib-arm_qemu-pnacl trybot, which uses qemu rather than
hardware. I updated the CL description to make this more clear.

   scons-stage-irt "arm" "${qemuflags}" "${SCONS_NONSFI}"
 }
 
 mode-buildbot-arm-hw() {
   FAIL_FAST=false
   local hwflags="-j2 -k naclsdk_validate=0 built_elsewhere=1"
 
   scons-stage-noirt "arm" "${hwflags}" "${SCONS_S_M}"
   # Large tests cannot be run in parallel
   scons-stage-noirt "arm" "${hwflags} -j1" "large_tests"
 
   # also run some tests with the irt
   scons-stage-irt "arm" "${hwflags}" "${SCONS_S_M_IRT}"
 
   scons-stage-noirt "arm" "${hwflags} pnacl_generate_pexe=0" "nonpexe_tests"
   scons-stage-irt "arm" \
     "${hwflags} use_sandboxed_translator=1 translate_in_build_step=0" \
     "toolchain_tests"
   scons-stage-irt "arm" \
     "${hwflags} use_sandboxed_translator=1 translate_fast=1 \
        translate_in_build_step=0" \
     "toolchain_tests"
 
   # Test Non-SFI Mode.
   scons-stage-irt "arm" "${hwflags}" "${SCONS_NONSFI_NEWLIB}"
   scons-stage-irt "arm" "${hwflags}" "${SCONS_NONSFI}"
 }
 
 mode-trybot-qemu() {
-  # Build and actually run the arm tests under qemu, except
-  # sandboxed translation. Hopefully that's a good tradeoff between
-  # flakiness and cycle time.
-  FAIL_FAST=false
-  local qemuflags="-j4 -k"
   clobber
+  # TODO(dschuff): move the gyp build to buildbot_pnacl.py
   gyp-arm-build
 
-  scons-stage-noirt "arm" "${qemuflags}" "${SCONS_EVERYTHING}"
-  # Large tests cannot be run in parallel
-  scons-stage-noirt "arm" "${qemuflags} -j1" "${SCONS_S_M} large_tests"
-
-  # also run some tests with the irt
-  scons-stage-irt "arm" "${qemuflags}" "${SCONS_S_M_IRT}"
-
-  # non-pexe tests
-  scons-stage-noirt "arm" "${qemuflags} pnacl_generate_pexe=0" "nonpexe_tests"
-
-  # Test Non-SFI Mode.
-  scons-stage-irt "arm" "${qemuflags}" "${SCONS_NONSFI_NEWLIB}"
-  scons-stage-irt "arm" "${qemuflags}" "${SCONS_NONSFI}"
+  ${BUILDBOT_PNACL_PY} opt arm pnacl
 }
 
 mode-buildbot-arm-dbg() {
   BUILD_MODE_HOST=DEDUG
   mode-buildbot-arm
   archive-for-hw-bots $(NAME_ARM_UPLOAD) regular
 }
 
 mode-buildbot-arm-opt() {
   mode-buildbot-arm
@@ skipping 26 matching lines @@
 # These 2 functions are also suitable for local TC sanity testing.
 tc-tests-all() {
   local is_try=$1
 
   local label="pnacl_newlib_dir=toolchain/${PNACL_TOOLCHAIN_DIR}"
   local scons_flags="-k skip_trusted_tests=1 -j8 ${label}"
 
   # newlib
   for arch in x86-32 x86-64 arm; do
     driver-tests "${arch}"
-    scons-stage-noirt "$arch" "${scons_flags}" "${SCONS_TC_TESTS}"
-    # Large tests cannot be run in parallel
-    scons-stage-noirt "$arch" "${scons_flags} -j1" "large_tests"
-    scons-stage-noirt "$arch" "${scons_flags} pnacl_generate_pexe=0" \
-        "nonpexe_tests"
   done
 
-  # Small set of sbtc tests w/ and without translate_fast=1.
-  scons-stage-irt "x86-32" "${scons_flags} use_sandboxed_translator=1" \
-    "toolchain_tests"
-  scons-stage-irt "x86-32" \
-    "${scons_flags} use_sandboxed_translator=1 translate_fast=1" \
-    "toolchain_tests"
-  scons-stage-irt "x86-64" "${scons_flags} use_sandboxed_translator=1" \
-    "toolchain_tests"
-  scons-stage-irt "x86-64" \
-    "${scons_flags} use_sandboxed_translator=1 translate_fast=1" \
-    "toolchain_tests"
-  # Smaller set of sbtc tests for ARM because qemu is flaky.
-  scons-stage-irt "arm" "${scons_flags} use_sandboxed_translator=1" \
-    "run_hello_world_test"
-  scons-stage-irt "arm" \
-    "${scons_flags} use_sandboxed_translator=1 translate_fast=1" \
-    "run_hello_world_test"
-
-  # Test Non-SFI Mode.
-  scons-stage-irt "x86-32" "${scons_flags}" "${SCONS_NONSFI}"
-  scons-stage-irt "arm" "${scons_flags}" "${SCONS_NONSFI}"
-
-  # Test unsandboxed mode.
-  scons-stage-irt "x86-32" "${scons_flags}" "pnacl_unsandboxed=1" \
-    "run_hello_world_test_irt"
+  # All the SCons tests (the same ones run by the main waterfall bot)
+  for arch in 32 64 arm; do
+    ${BUILDBOT_PNACL_PY} opt "${arch}" pnacl
+  done
 
   # Run the GCC torture tests just for x86-32.  Testing a single
   # architecture gives good coverage without taking too long.  We
   # don't test x86-64 here because some of the torture tests fail on
   # the x86-64 toolchain trybot (though not the buildbots, apparently
   # due to a hardware difference:
   # https://code.google.com/p/nativeclient/issues/detail?id=3697).
+
   # Build the SDK libs first so that linking will succeed.
   echo "@@@BUILD_STEP sdk libs @@@"
   ${PNACL_BUILD} sdk
+  ./scons --verbose platform=x86-32 -j8 sel_ldr

[Mark Seaborn, 2014/07/29 17:11:33]

I'm curious: How come this isn't covered by BUILDBOT_PNACL_PY earlier?  Does
that build the dbg version?

[Derek Schuff, 2014/07/29 18:41:50]

It's because the python version clobbers all the scons output directories (for
all architectures) when it starts. So when the 64 bit tests run it clobbers the
32 bit sel_ldr that was there.

 
   echo "@@@BUILD_STEP torture_tests x86-32 @@@"
   tools/toolchain_tester/torture_test.py pnacl x86-32 --verbose \
       --concurrency=8 || handle-error
 }
 
 tc-tests-fast() {
   local arch="$1"
   local scons_flags="-k skip_trusted_tests=1"
 
@@ skipping 54 matching lines @@
   exit 1
 fi
 
 "$@"
 
 if [[ ${RETCODE} != 0 ]]; then
   echo "@@@BUILD_STEP summary@@@"
   echo There were failed stages.
   exit ${RETCODE}
 fi