Add more robust ssh ping to be run on reboot for updates.

remote_access.sh

 # Copyright (c) 2009 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 # Library for setting up remote access and running remote commands.
 
 DEFAULT_PRIVATE_KEY="${GCLIENT_ROOT}/src/scripts/mod_for_test_scripts/\
 ssh_keys/testing_rsa"
 
 DEFINE_string remote "" "remote hostname/IP of running Chromium OS instance"
@@ skipping 16 matching lines @@
             UserKnownHostsFile=$TMP_KNOWN_HOSTS" --no-R \
     --files-from=$1 root@${FLAGS_remote}:/ $2
 }
 
 function remote_sh() {
   REMOTE_OUT=$(ssh -p ${FLAGS_ssh_port} -o StrictHostKeyChecking=no -o \
     UserKnownHostsFile=$TMP_KNOWN_HOSTS root@$FLAGS_remote "$@")
   return ${PIPESTATUS[0]}
 }
 
+# Robust ping that will monitor ssh and not hang even if ssh hangs.
+# This method attempts to run true on the remote device.  We check to see
+# if ssh is still running after a timeout (is hanging).  If so, we retry
+# 3 times before giving up.  We wait up to 5 seconds for true to return each
+# time.
+function ping_ssh() {
+  local timeout=5
+  local timeout_inc=.5
+  local retries=3
+  while [ ${retries} -gt 0 ]; do
+    nohup -- remote_sh "true" 2> /dev/null &
+    local ssh_pid=$!
+    local current_timeout=0

[petkov, 2011/01/12 18:38:52]

this site shows an interesting way to implement a timeout that doesn't require
waiting for the full timeout yet doesn't do a loop:
http://redflo.de/tiki-index.php?page=Bash+script+with+timeout+function

+    while ps -p ${ssh_pid} > /dev/null && \

[petkov, 2011/01/12 18:38:52]

no need for trailing \ I think

+        [ ${current_timeout} -gt ${timeout} ]; do

[petkov, 2011/01/12 18:38:52]

current_timeout is 0, timeout is 5 -- this evaluates to false, no? so the loop
will never iterate?

+      sleep ${timeout}
+      echo "blah"

[petkov, 2011/01/12 18:38:52]

blah? :)

+      current_timeout=$((current_timeout+timeout_inc))

[petkov, 2011/01/12 18:38:52]

add spaces around +

+    done
+    # If pid has been killed, kill will return 0.
+    if ! kill ${ssh_pid} &> /dev/null; then
+      break
+    else

[petkov, 2011/01/12 18:38:52]

s/else/fi/, or better:

kill ... || break

are you sure kill SIGTERM won't hang? should you just kill -9 instead?

+      retries=$((retries-1))
+    fi
+  done
+}
+
 function remote_sh_allow_changed_host_key() {
   rm -f $TMP_KNOWN_HOSTS
+  # Often on first boot we can have a race condition where the initial ssh
+  # can hang.  Pinging first mitigates this.
+  ping_ssh
   remote_sh "$@"
 }
 
 function set_up_remote_access() {
   if [ -z "$SSH_AGENT_PID" ]; then
     eval $(ssh-agent)
     OWN_SSH_AGENT=1
   else
     OWN_SSH_AGENT=0
   fi
@@ skipping 57 matching lines @@
 
 function remote_access_init() {
   TMP_PRIVATE_KEY=$TMP/private_key
   TMP_KNOWN_HOSTS=$TMP/known_hosts
   if [ -z "$FLAGS_remote" ]; then
     echo "Please specify --remote=<IP-or-hostname> of the Chromium OS instance"
     exit 1
   fi
   set_up_remote_access
 }