Adding timeout for log server access + more safety with bad response....

command_wrapper/bin/command_wrapper.py

Index: command_wrapper/bin/command_wrapper.py
===================================================================
--- command_wrapper/bin/command_wrapper.py	(revision 54521)
+++ command_wrapper/bin/command_wrapper.py	(working copy)
@@ -15,13 +15,18 @@
 import optparse
 import os
 import platform
+import socket
 import subprocess
 import sys
+import threading
 import time
 import urllib
 import uuid
 
 
+LOG_TIMEOUT = 10
+
+
 def LogCommand(options, command_id,
                attempt, cmd, returncode, stdout, stderr, runtime):
   """Log a command invocation and result to a central location.
@@ -60,9 +65,22 @@
   f = urllib.urlopen(options.logurl, params)
   ret = f.read()
   f.close()
-  return int(ret) != 0
+  try:
+    return int(ret) != 0
+  except ValueError:
+    return 0
 
 
+def RunWithTimeout(timeout, func, *args, **kwargs):
+  result = None
+  def CallFunc():
+    result = func(*args, **kwargs)
+  th = threading.Thread(target=CallFunc)
+  th.start()
+  th.join(timeout)
+  return result
+
+
 def main(argv):
   parser = optparse.OptionParser()
   parser.add_option('-r', '--retries', dest='retries',
@@ -73,8 +91,17 @@
                     help='URL to log invocations/failures to')
   (options, args) = parser.parse_args(args=argv[1:])
 
+  # Limit tcp connnection timeouts to 10 seconds.
+  socket.setdefaulttimeout(10)
+
   command_id = uuid.uuid1()
   cmd = ' '.join(args)
+
+  # Log that we're even starting.
+  RunWithTimeout(LOG_TIMEOUT, LogCommand,
+                 options, command_id, -1, cmd, -1, '', '', 0)
+
+  # Try up to a certain number of times.
   for r in range(options.retries):
     tm = time.time()
     p = subprocess.Popen(cmd, shell=True,
@@ -84,8 +111,9 @@
     sys.stdout.write(p_stdout)
     sys.stderr.write(p_stderr)
     runtime = time.time() - tm
-    accept = LogCommand(options, command_id, r, cmd,
-                        p.returncode, p_stdout, p_stderr, runtime)
+    accept = RunWithTimeout(LOG_TIMEOUT, LogCommand,
+                            options, command_id, r, cmd,
+                            p.returncode, p_stdout, p_stderr, runtime)
     if accept:
       return p.returncode
     if p.returncode == 0: