Fix silent failures in test runner that lead to long timeouts

build/android/pylib/cmd_helper.py

Index: build/android/pylib/cmd_helper.py
diff --git a/build/android/pylib/cmd_helper.py b/build/android/pylib/cmd_helper.py
index 4aa17b9d10726afd14cbd14eb2227aa28894cc41..aba00be735368e9e931e321b14bef1d46ad1fbf3 100644
--- a/build/android/pylib/cmd_helper.py
+++ b/build/android/pylib/cmd_helper.py
@@ -5,12 +5,19 @@
 """A wrapper for subprocess to make calling shell commands easier."""
 
 import logging
+import os
 import pipes
+import select
 import signal
+import StringIO
 import subprocess
-import tempfile
+import time
 
-from pylib.utils import timeout_retry
+# fcntl is not available on Windows.
+try:
+  import fcntl
+except ImportError:
+  fcntl = None
 
 
 def Popen(args, stdout=None, stderr=None, shell=None, cwd=None, env=None):
@@ -88,33 +95,73 @@ def GetCmdStatusAndOutput(args, cwd=None, shell=False):
     s += ':' + cwd
   s += '> ' + args_repr
   logging.info(s)
-  tmpout = tempfile.TemporaryFile(bufsize=0)
-  tmperr = tempfile.TemporaryFile(bufsize=0)
-  exit_code = Call(args, cwd=cwd, stdout=tmpout, stderr=tmperr, shell=shell)
-  tmperr.seek(0)
-  stderr = tmperr.read()
-  tmperr.close()
+  pipe = Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+               shell=shell, cwd=cwd)
+  stdout, stderr = pipe.communicate()
+
   if stderr:
     logging.critical(stderr)
-  tmpout.seek(0)
-  stdout = tmpout.read()
-  tmpout.close()
   if len(stdout) > 4096:
     logging.debug('Truncated output:')
   logging.debug(stdout[:4096])
-  return (exit_code, stdout)
+  return (pipe.returncode, stdout)
+
+
+class TimeoutError(Exception):
+  """Module-specific timeout exception."""
+  pass
 
 
-def GetCmdStatusAndOutputWithTimeoutAndRetries(args, timeout, retries):
-  """Executes a subprocess with a timeout and retries.
+def GetCmdStatusAndOutputWithTimeout(args, timeout, cwd=None, shell=False,
+                                     logfile=None):
+  """Executes a subprocess with a timeout.
 
   Args:
     args: List of arguments to the program, the program to execute is the first
       element.
-    timeout: the timeout in seconds.
-    retries: the number of retries.
+    timeout: the timeout in seconds or None to wait forever.
+    cwd: If not None, the subprocess's current directory will be changed to
+      |cwd| before it's executed.
+    shell: Whether to execute args as a shell command.
+    logfile: Optional file-like object that will receive output from the
+      command as it is running.
 
   Returns:
     The 2-tuple (exit code, output).
   """
-  return timeout_retry.Run(GetCmdStatusAndOutput, timeout, retries, [args])
+  assert fcntl, 'fcntl module is required'
+  process = Popen(args, cwd=cwd, shell=shell, stdout=subprocess.PIPE,
+                  stderr=subprocess.STDOUT)
+  try:
+    end_time = (time.time() + timeout) if timeout else None
+    poll_interval = 1
+    buffer_size = 4096
+    child_fd = process.stdout.fileno()
+    output = StringIO.StringIO()
+
+    # Enable non-blocking reads from the child's stdout.
+    fl = fcntl.fcntl(child_fd, fcntl.F_GETFL)
+    fcntl.fcntl(child_fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
+
+    while True:
+      if end_time and time.time() > end_time:
+        raise TimeoutError
+      read_fds, _, _ = select.select([child_fd], [], [], poll_interval)
+      if child_fd in read_fds:
+        data = os.read(child_fd, buffer_size)
+        if not data:
+          break
+        if logfile:
+          logfile.write(data)
+        output.write(data)
+      if process.poll() is not None:
+        break
+  finally:
+    try:
+      # Make sure the process doesn't stick around if we fail with an
+      # exception.
+      process.kill()
+    except OSError:
+      pass
+    process.wait()
+  return process.returncode, output.getvalue()