add and enable validator tests for x86

buildbot/buildbot_spec2k.sh

 #!/bin/bash
 # Copyright (c) 2012 The Native Client Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 set -o xtrace
 set -o nounset
 set -o errexit
 
 ######################################################################
 # SCRIPT CONFIG
 ######################################################################
 
-CLOBBER=${CLOBBER:-yes}
-SCONS_TRUSTED="./scons --mode=opt-host -j8"
-SCONS_NACL="./scons --mode=opt-host,nacl -j8"
-SPEC_HARNESS=${SPEC_HARNESS:-${HOME}/cpu2000-redhat64-ia32}/
+readonly CLOBBER=${CLOBBER:-yes}
+readonly SCONS_TRUSTED="./scons --mode=opt-host -j8"
+readonly SCONS_NACL="./scons --mode=opt-host,nacl -j8"
+readonly SPEC_HARNESS=${SPEC_HARNESS:-${HOME}/cpu2000-redhat64-ia32}/
 
-TRYBOT_TESTS="176.gcc 179.art 181.mcf 197.parser 252.eon 254.gap"
-TRYBOT_TRANSLATOR_TESTS="176.gcc"
-TRYBOT_X86_64_ZERO_BASED_SANDBOX_TESTS="176.gcc"
+readonly TRYBOT_TESTS="176.gcc 179.art 181.mcf 197.parser 252.eon 254.gap"
+readonly TRYBOT_TRANSLATOR_TESTS="176.gcc"
+readonly TRYBOT_X86_64_ZERO_BASED_SANDBOX_TESTS="176.gcc"
 
 readonly BUILDBOT_PNACL="buildbot/buildbot_pnacl.sh"
 readonly UP_DOWN_LOAD="buildbot/file_up_down_load.sh"
 
-SPEC_BASE="tests/spec2k"
+readonly SPEC_BASE="tests/spec2k"
 readonly ARCHIVE_NAME=$(${SPEC_BASE}/run_all.sh GetTestArchiveName)
 
 readonly NAME_ARM_TRY_UPLOAD=$(${BUILDBOT_PNACL} NAME_ARM_TRY_UPLOAD)
 readonly NAME_ARM_TRY_DOWNLOAD=$(${BUILDBOT_PNACL} NAME_ARM_TRY_DOWNLOAD)
 readonly NAME_ARM_UPLOAD=$(${BUILDBOT_PNACL} NAME_ARM_UPLOAD)
 readonly NAME_ARM_DOWNLOAD=$(${BUILDBOT_PNACL} NAME_ARM_DOWNLOAD)
 
+readonly QEMU_TOOL="$(pwd)/toolchain/linux_arm-trusted/run_under_qemu_arm"
+
+readonly CANNED_NEXE_REV=1001
+
 # If true, terminate script when first error is encountered.
-FAIL_FAST=${FAIL_FAST:-false}
+readonly FAIL_FAST=${FAIL_FAST:-false}
 RETCODE=0
 
 # Print the number of tests being run for the buildbot status output
 testcount() {
   local tests="$1"
   if [[ ${tests} == "all" ]]; then
     echo "all"
   else
     echo ${tests} | wc -w
   fi
@@ skipping 98 matching lines @@
   else
     ${UP_DOWN_LOAD} DownloadArmBinariesForHWBots ${NAME_ARM_DOWNLOAD} \
       ${ARCHIVE_NAME}
   fi
   echo "@@@BUILD_STEP spec2k untar@@@"
   pushd ${SPEC_BASE}
   ./run_all.sh UnpackArmBinaries
   popd
 }
 
+download-validator-test-nexes() {
+  local arch="$1"
+  echo "@@@BUILD_STEP validator test download@@@"
+  ${UP_DOWN_LOAD} DownloadArchivedNexes ${CANNED_NEXE_REV} \

[jvoung (off chromium), 2012/11/15 19:10:48]

Did you want to reuse tools/canned_nexe_tool.sh instead?

+      "${arch}_giant" giant_nexe.tar.bz2
+  # This generates "CannedNexes/" in the current directory
+ rm -rf CannedNexes

[jvoung (off chromium), 2012/11/15 19:10:48]

indentation

[robertm, 2012/11/15 19:33:31]

Done.

+ tar jxf  giant_nexe.tar.bz2
+}
+
+get-validator() {
+  local arch="$1"
+  if [[ ${arch} == "x86-32" ]] ; then
+      echo "$(pwd)/scons-out/opt-linux-x86-32/staging/ncval"

[jvoung (off chromium), 2012/11/15 19:14:19]

Also, would the dfa-validator devs be interested in using this?  I think they
had some scripts to run benchmarks offline, but not actually get them benchmarks
tracked on the bots.

[robertm, 2012/11/15 19:33:31]

absolutely, will send email to khim about it

On 2012/11/15 19:14:19, jvoung (chromium) wrote:

+  elif [[ ${arch} == "x86-64" ]] ; then
+      echo "$(pwd)/scons-out/opt-linux-x86-64/staging/ncval"
+  elif [[ ${arch} == "arm" ]] ; then
+      echo "$(pwd)/scons-out/opt-linux-arm/staging/arm-ncval-core"
+  else
+      echo "ERROR: unknown arch"
+  fi
+}
+
+LogTimeHelper() {
+  # This format is recognized by the buildbot system
+  echo "RESULT $1_$2: $3= $(bc) secs"

[jvoung (off chromium), 2012/11/15 19:10:48]

What does the invocation of bc do here?
Oh... I see from below.

Why don't you compute the value within LogTimedRun, then just pass the value
into LogTimeHelper instead of splitting the calculation up between the two
functions?

[robertm, 2012/11/15 19:33:31]

I eliminated the mess with the temp files.
so the output of /usr/bin/time is piped straight to
LogTimeHelper that means that LogTimedRun does not
see the output of "time".

On 2012/11/15 19:10:48, jvoung (chromium) wrote:

+}
+
+LogTimedRun() {
+  local graph=$1
+  local benchmark=$2
+  local variant=$3
+  shift 3
+  # S: system mode CPU-seconds used by the process
+  # U: user mode CPU-seconds  used by the process
+  # We add a plus sign inbetween so that we can pipe the output to "bc"
+  /usr/bin/time -f "%U + %S" \
+      --output >(LogTimeHelper ${graph} ${benchmark} ${variant}) \

[jvoung (off chromium), 2012/11/15 21:00:43]

Does --output need an argument?

[robertm, 2012/11/15 21:12:37]

added a comment

On 2012/11/15 21:00:43, jvoung (chromium) wrote:

+      "$@"
+}
+
+measure-validator-speed() {

[jvoung (off chromium), 2012/11/15 21:00:43]

Perhaps add a cross reference to the canned-nexe tool to indicate how to
regenerate the nexes if validation rules change.

[robertm, 2012/11/15 21:12:37]

I added a comment before:
readonly CANNED_NEXE_REV=1001

On 2012/11/15 21:00:43, jvoung (chromium) wrote:

+  local arch="$1"
+  local validator=$(get-validator ${arch})
+  echo "@@@BUILD_STEP validator speed test@@@"

[jvoung (off chromium), 2012/11/15 19:10:48]

maybe print the ${arch} too inside the BUILD_STEP

[robertm, 2012/11/15 19:33:31]

Done.

+  if [[ ! -e ${validator} ]] ; then
+    echo "ERROR: missing validator executable: ${validator}"
+    handle-error
+    return
+  fi
+
+  if [[ ${arch} == "arm" && $(uname -p) != arm* ]] ; then
+    validator="${QEMU_TOOL} ${validator}"

[jvoung (off chromium), 2012/11/15 19:10:48]

Do we really want to run this in qemu, or can we do this on the hardware bot
only, as "pnacl-trybot-arm-hw" does?

Maybe add the step to pnacl-arm-hw too.

[robertm, 2012/11/15 19:33:31]

The qemu part is useful when I manually tests this.
I definitely want to enable this for arm-hw 
but I have some feeling it might break or could be flake.
So, I first want to get it in for x86 and make sure it works
there which is more predictable.


On 2012/11/15 19:10:48, jvoung (chromium) wrote:

+  fi
+
+  for nexe in CannedNexes/* ; do
+    ls --size --block-size=1 ${nexe}
+    LogTimedRun  "validationtime" $(basename ${nexe}) "canned" \
+        ${validator} ${nexe}
+    echo "next"

[jvoung (off chromium), 2012/11/15 19:10:48]

why not echo "validating ${nexe}" at the beginning, instead of echo'ing "next"

[robertm, 2012/11/15 19:33:31]

that was debugging leftovers - remvoed

+  done
+}
+
 ######################################################################
 # NOTE: trybots only runs a subset of the the spec2k tests
 # TODO: elminate this long running bot in favor per arch sharded bots
 pnacl-trybot-arm-qemu() {
   clobber
   build-prerequisites "arm" "bitcode"
   build-tests SetupPnaclArmOpt "${TRYBOT_TESTS}" 0 1
   run-tests SetupPnaclArmOpt "${TRYBOT_TESTS}" 0 1
 }
 
@@ skipping 16 matching lines @@
     handle-error
   popd
 }
 
 pnacl-trybot-x8632() {
   clobber
   build-prerequisites "x86-32" "bitcode"
   build-tests SetupPnaclX8632Opt "${TRYBOT_TESTS}" 0 1
   run-tests SetupPnaclX8632Opt "${TRYBOT_TESTS}" 0 1
   build-tests SetupPnaclTranslatorX8632Opt "${TRYBOT_TRANSLATOR_TESTS}" 0 1
   run-tests SetupPnaclTranslatorX8632Opt "${TRYBOT_TRANSLATOR_TESTS}" 0 1

[jvoung (off chromium), 2012/11/15 21:02:04]

Perhaps add it to the trybot runs too, so that we can see what the output would
have been.  It's safe in that it won't create superfluous graphs or tracked data
points, since the trybots don't have the tracking enabled.

[robertm, 2012/11/15 21:12:37]

Done. for x86

 }
 
 pnacl-x86-64-zero-based-sandbox() {
   clobber
   export NACL_ENABLE_INSECURE_ZERO_BASED_SANDBOX=1
   build-prerequisites "x86-64" "bitcode" "x86_64_zero_based_sandbox=1"
   build-tests SetupPnaclX8664ZBSOpt \
     "${TRYBOT_X86_64_ZERO_BASED_SANDBOX_TESTS}" 0 1
   run-tests SetupPnaclX8664ZBSOpt \
     "${TRYBOT_X86_64_ZERO_BASED_SANDBOX_TESTS}" 0 1
@@ skipping 38 matching lines @@
 pnacl-x8664() {
   clobber
   build-prerequisites "x86-64" "bitcode"
   local setups="SetupPnaclX8664 \
                SetupPnaclX8664Opt \
                SetupPnaclTranslatorX8664 \
                SetupPnaclTranslatorX8664Opt"
   build-tests "${setups}" all 1 3
   run-tests "${setups}" all 1 3
   pnacl-x86-64-zero-based-sandbox
+  measure-validator-speed x86-64
 }
 
 pnacl-x8632() {
   clobber
   build-prerequisites "x86-32" "bitcode"
   local setups="SetupPnaclX8632 \
                 SetupPnaclX8632Opt \
                 SetupPnaclTranslatorX8632 \
                 SetupPnaclTranslatorX8632Opt"
   build-tests "${setups}" all 1 3
   run-tests "${setups}" all 1 3
+  measure-validator-speed x86-32
 }
 
 nacl-x8632() {
   clobber
   build-prerequisites "x86-32" ""
   local setups="SetupNaclX8632 \
                 SetupNaclX8632Opt"
   build-tests "${setups}" all 1 3
   run-tests "${setups}" all 1 3
+  measure-validator-speed x86-32
 }
 
 nacl-x8664() {
   clobber
   build-prerequisites "x86-64" ""
   local setups="SetupNaclX8664 \
                 SetupNaclX8664Opt"
   build-tests "${setups}" all 1 3
   run-tests "${setups}" all 1 3
+  measure-validator-speed x86-64
 }
 
 
 ######################################################################
 # Script assumed to be run in native_client/
 if [[ $(pwd) != */native_client ]]; then
   echo "ERROR: must be run in native_client!"
   exit 1
 fi
 
 
 if [[ $# -eq 0 ]] ; then
   echo "you must specify a mode on the commandline:"
   exit 1
 fi
 
 if [ "$(type -t $1)" != "function" ]; then
   Usage
   echo "ERROR: unknown mode '$1'." >&2
   exit 1
 fi
 
 "$@"
 
 if [[ ${RETCODE} != 0 ]]; then
   echo "@@@BUILD_STEP summary@@@"
   echo There were failed stages.
   exit ${RETCODE}
 fi