Improvements for to continuous test loop.

src/scripts/run_remote_tests.sh

 #!/bin/bash
 
 # Copyright (c) 2009 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 # Script to run client or server tests on a live remote image.
  
 # Load common constants.  This should be the first executable line.
 # The path to common.sh should be relative to your script's location.
 
 . "$(dirname $0)/common.sh"
 . "$(dirname $0)/remote_access.sh"
 
 DEFAULT_OUTPUT_FILE=test-output-$(date '+%Y%m%d.%H%M%S')
 
 DEFINE_boolean cleanup ${FLAGS_TRUE} "Clean up temp directory"
 DEFINE_integer iterations 1 "Iterations to run every top level test" i
 DEFINE_string output_file "${DEFAULT_OUTPUT_FILE}" "Test run output" o
 DEFINE_boolean verbose ${FLAGS_FALSE} "Show verbose autoserv output" v
 DEFINE_boolean update_db ${FLAGS_FALSE} "Put results in autotest database" u
 DEFINE_string machine_desc "" "Machine description used in database"
 DEFINE_string build_desc "" "Build description used in database"
+DEFINE_string results_dir_root "" \
+    "Directory to place individual autoserv results files (default to TMP)"
 
 function cleanup() {
   if [[ $FLAGS_cleanup -eq ${FLAGS_TRUE} ]]; then
     rm -rf "${TMP}"
   else
     echo "Left temporary files at ${TMP}"
   fi
 }
 
 # Returns an error if the test_result_file has text which indicates
@@ skipping 62 matching lines @@
   if [[ ${FLAGS_update_db} -eq ${FLAGS_TRUE} && ! -x "${parse_cmd}" ]]; then
     echo "Cannot find parser ${parse_cmd}"
     exit 1
   fi
 
   set -e
   
   # Set global TMP for remote_access.sh's sake
   TMP=$(mktemp -d /tmp/run_remote_tests.XXXX)
 
+  if [[ -z "${FLAGS_results_dir_root}" ]]; then
+    FLAGS_results_dir_root="${TMP}"
+  fi
+
   rm -f "${FLAGS_output_file}"
 
   trap cleanup EXIT
 
   cp -r "${SRC_ROOT}/third_party/autotest/files" "${TMP}/autotest"
   
   local control_files_to_run=""
+  local any_failures=0
 
   # Now search for tests which unambiguously include the given identifier
   local search_path=$(echo ${TMP}/autotest/{client,server}/{tests,site_tests})
   for test_request in $FLAGS_ARGV; do
     test_request=$(remove_quotes "${test_request}")
     ! finds=$(find ${search_path} -type f -name control | \
       egrep "${test_request}")
     if [[ -z "${finds}" ]]; then
       echo "Can not find match for ${test_request}"
-      exit 1
+      any_failures=1
+      continue
     fi
     local matches=$(echo "${finds}" | wc -l)
     if [[ ${matches} -gt 1 ]]; then
       echo "${test_request} is ambiguous:"
       echo "${finds}"
-      exit 1
+      any_failures=1
+      continue
     fi
     for i in $(seq 1 $FLAGS_iterations); do
       control_files_to_run="${control_files_to_run} '${finds}'"
     done
   done
 
   echo "Running the following control files: ${control_files_to_run}"
 
   remote_access_init
   
@@ skipping 14 matching lines @@
       option="-c"
     elif [ "${type}" == "server" ]; then
      option="-s"
     else
       echo "Unknown type of test (${type}) in ${control_file}"
       exit 1
     fi
     echo "Running ${type} test ${control_file}"
     local short_name=$(basename $(dirname "${control_file}"))
     local start_time=$(date '+%s')
-    local results_dir="${TMP}/${short_name},${FLAGS_machine_desc},${start_time}"
+    local results_dir_name="${short_name},${FLAGS_machine_desc},${start_time}"
+    local results_dir="${FLAGS_results_dir_root}/${results_dir_name}"
     rm -rf "${results_dir}"
     local verbose=""
     if [[ ${FLAGS_verbose} -eq $FLAGS_TRUE ]]; then
       verbose="--verbose"
     fi
-    ${autoserv} -m "${FLAGS_remote}" "${option}" "${control_file}" \
-      -r "${results_dir}" ${verbose}
+    if ! ${autoserv} -m "${FLAGS_remote}" "${option}" "${control_file}" \
+      -r "${results_dir}" ${verbose}; then
+      echo "Autoserv run of ${control_file} failed." | \
+        tee -a "${FLAGS_output_file}"
+      any_failures=1
+      continue
+    fi
     local test_status="${results_dir}/status"
     local test_result_dir="${results_dir}/${short_name}"
     local keyval_file="${test_result_dir}/results/keyval"
     if is_successful_test "${test_status}"; then
       echo "${control_file} succeeded." | tee -a "${FLAGS_output_file}"
       if [[ -f "${keyval_file}" ]]; then
         echo "Keyval was:" | tee -a "${FLAGS_output_file}"
         cat "${keyval_file}" | tee -a "${FLAGS_output_file}"
       fi
     else
@@ skipping 11 matching lines @@
       add_test_attribute "${results_dir}" server-start-time "${start_time}"
       add_test_attribute "${results_dir}" server-end-time "${end_time}"
       if ! "${parse_cmd}" -o "${results_dir}"; then
         echo "Parse failed." | tee -a "${FLAGS_output_file}"
         FLAGS_cleanup=${FLAGS_FALSE}
       fi
     fi
   done
 
   echo "Output stored to ${FLAGS_output_file}"
+
+  return ${any_failures}
 }
 
 main $@