Revert remote_access refactor.

remote_access.sh

Index: remote_access.sh
diff --git a/remote_access.sh b/remote_access.sh
index c7152cc832e5a6f3b3c05812dfe029ad631052dc..121d74905888a065a47265373afaab9cd9ddaf0d 100644
--- a/remote_access.sh
+++ b/remote_access.sh
@@ -16,8 +16,7 @@ DEFINE_integer ssh_port 22 \
 # Copies $1 to $2 on remote host
 function remote_cp_to() {
   REMOTE_OUT=$(scp -P ${FLAGS_ssh_port} -o StrictHostKeyChecking=no \
-    -o ServerAliveInterval=60 -o UserKnownHostsFile=$TMP_KNOWN_HOSTS \
-    -i $TMP_PRIVATE_KEY $1 \
+    -o UserKnownHostsFile=$TMP_KNOWN_HOSTS -i $TMP_PRIVATE_KEY $1 \
     root@$FLAGS_remote:$2)
   return ${PIPESTATUS[0]}
 }
@@ -26,15 +25,13 @@ function remote_cp_to() {
 # $2.  Directory paths in $1 are collapsed into $2.
 function remote_rsync_from() {
   rsync -e "ssh -p ${FLAGS_ssh_port} -o StrictHostKeyChecking=no \
-             -o ServerAliveInterval=60 -o UserKnownHostsFile=$TMP_KNOWN_HOSTS \
-             -i $TMP_PRIVATE_KEY" \
+             -o UserKnownHostsFile=$TMP_KNOWN_HOSTS -i $TMP_PRIVATE_KEY" \
     --no-R --files-from=$1 root@${FLAGS_remote}:/ $2
 }
 
 function remote_sh() {
   REMOTE_OUT=$(ssh -p ${FLAGS_ssh_port} -o StrictHostKeyChecking=no \
     -o UserKnownHostsFile=$TMP_KNOWN_HOSTS -i $TMP_PRIVATE_KEY \
-    -o ServerAliveInterval=60 \
     root@$FLAGS_remote "$@")
   return ${PIPESTATUS[0]}
 }
@@ -77,12 +74,29 @@ function learn_arch() {
   info "Target reports arch is ${FLAGS_arch}"
 }
 
-function remote_reboot {
-  info "Rebooting."
-  remote_sh "touch /tmp/awaiting_reboot; reboot"
-  local output_file
-  output_file="${TMP}/output"
+# Checks to see if pid $1 is running.
+function is_pid_running() {
+  ps -p ${1} 2>&1 > /dev/null
+}
+
+# Wait function given an additional timeout argument.
+# $1 - pid to wait on.
+# $2 - timeout to wait for.
+function wait_with_timeout() {
+  local pid=$1
+  local timeout=$2
+  local -r TIMEOUT_INC=1
+  local current_timeout=0
+  while is_pid_running ${pid} && [ ${current_timeout} -lt ${timeout} ]; do
+    sleep ${TIMEOUT_INC}
+    current_timeout=$((current_timeout + TIMEOUT_INC))
+  done
+  ! is_pid_running ${pid}
+}
 
+# Checks to see if a machine has rebooted using the presence of a tmp file.
+function check_if_rebooted() {
+  local output_file="${TMP}/output"
   while true; do
     REMOTE_OUT=""
     # This may fail while the machine is down so generate output and a
@@ -93,12 +107,23 @@ function remote_reboot {
     if grep -q "0" "${output_file}"; then
       if grep -q "1" "${output_file}"; then
         info "Not yet rebooted"
+        sleep .5
       else
         info "Rebooted and responding"
         break
       fi
     fi
-    sleep .5
+  done
+}
+
+function remote_reboot() {
+  info "Rebooting."
+  remote_sh "touch /tmp/awaiting_reboot; reboot"
+  while true; do
+    check_if_rebooted &
+    local pid=$!
+    wait_with_timeout ${pid} 30 && break
+    ! kill -9 ${pid} 2> /dev/null
   done
 }