Add ability to vm library to retry until we can ssh into it.

lib/cros_vm_lib.sh

Index: lib/cros_vm_lib.sh
diff --git a/lib/cros_vm_lib.sh b/lib/cros_vm_lib.sh
index 1425f24d6cb33f40205a4be3446e57f8a6399c2a..8f98926ccc9e8de2ebd5559ee8d1756fb79640a9 100644
--- a/lib/cros_vm_lib.sh
+++ b/lib/cros_vm_lib.sh
@@ -12,6 +12,7 @@ DEFINE_boolean snapshot ${FLAGS_FALSE} "Don't commit changes to image."
 DEFINE_integer ssh_port 9222 "Port to tunnel ssh traffic over."
 
 KVM_PID_FILE=/tmp/kvm.$$.pid
+LIVE_VM_IMAGE=
 
 function get_pid() {
   sudo cat "${KVM_PID_FILE}"
@@ -55,18 +56,49 @@ function start_kvm() {
       ${snapshot} \
       -net user,hostfwd=tcp::${FLAGS_ssh_port}-:22 \
       -hda "${1}"
+
+    LIVE_VM_IMAGE="${1}"
   fi
 }
 
+# Checks to see if we can access the target virtual machine with ssh.
+function ssh_ping() {
+  "$(dirname $0)"/../ssh_test.sh \
+    --ssh_port=${FLAGS_ssh_port} \
+    --remote=127.0.0.1
+}
+
+# Tries to ssh into live image $1 times.  After first failure, a try involves
+# shutting down and restarting kvm.
+function retry_until_ssh() {
+  local can_ssh_into=1
+  local retries=0
+  ssh_ping && can_ssh_into=0
+
+  while [ ${can_ssh_into} -eq 1 ] && [ ${retries} -lt ${1} ]; do
+    echo "Failed to connect to virtual machine, retrying ... " >&2
+    stop_kvm || echo "Could not stop kvm.  Retrying anyway." >&2
+    start_kvm "${LIVE_VM_IMAGE}"
+    ssh_ping && can_ssh_into=0
+    retries=$((retries + 1))
+  done
+  return ${can_ssh_into}
+}
+
 function stop_kvm() {
   if [ "${FLAGS_persist}" -eq "${FLAGS_TRUE}" ]; then
     echo "Persist requested.  Use --ssh_port ${FLAGS_ssh_port} " \
       "--kvm_pid ${KVM_PID_FILE} to re-connect to it."
   else
-    echo "Stopping the KVM instance"
+    echo "Stopping the KVM instance" >&2
     local pid=$(get_pid)
-    echo "Killing ${pid}"
-    sudo kill ${pid}
-    sudo rm "${KVM_PID_FILE}"
+    if [ -n "${pid}" ]; then
+      echo "Killing ${pid}"
+      sudo kill ${pid}
+      sudo rm "${KVM_PID_FILE}"
+    else
+      echo "No kvm pid found to stop." >&2
+      return 1
+    fi
   fi
 }