Implement RunSubprocessChain to allow the tools call several processes connected by pipes.

tools/valgrind/common.py

Index: tools/valgrind/common.py
===================================================================
--- tools/valgrind/common.py	(revision 100116)
+++ tools/valgrind/common.py	(working copy)
@@ -33,23 +33,29 @@
   return subprocess.Popen(proc)
 
 
-def RunSubprocess(proc, timeout=0, detach=False, background=False):
-  """ Runs a subprocess, until it finishes or |timeout| is exceeded and the
-  process is killed with taskkill.  A |timeout| <= 0  means no timeout.
+def RunSubprocessChain(procs, timeout=0, detach=False, background=False):
+  """ Runs a chain of subprocesses, connected by a pipe until it finishes
+  or |timeout| is exceeded and the processes are killed with taskkill.
+  A |timeout| <= 0  means no timeout.
 
   Args:
-    proc: list of process components (exe + args)
+    procs: list of commands to execute (each is a list of exe + args).
     timeout: how long to wait before killing, <= 0 means wait forever
     detach: Whether to pass the DETACHED_PROCESS argument to CreateProcess
         on Windows.  This is used by Purify subprocesses on buildbot which
         seem to get confused by the parent console that buildbot sets up.
   """
-
-  logging.info("running %s, timeout %d sec" % (" ".join(proc), timeout))
+  command_str = " | ".join(map(lambda l: " ".join(l), procs))
+  logging.info("running %s, timeout %d sec" % (command_str, timeout))
+  processes = []
   if detach:
+    if len(procs) > 1:
+      raise NotImplementedError("Chaining of detachable processes "
+                                "is not supported yet.")
     # see MSDN docs for "Process Creation Flags"
     DETACHED_PROCESS = 0x8
-    p = subprocess.Popen(proc, creationflags=DETACHED_PROCESS)
+    processes.append(subprocess.Popen(procs[0],
+                                      creationflags=DETACHED_PROCESS))
   else:
     # For non-detached processes, manually read and print out stdout and stderr.
     # By default, the subprocess is supposed to inherit these from its parent,
@@ -58,8 +64,23 @@
     # it came from us for buildbot to read it.  We're not sure why this is
     # necessary.
     # TODO(erikkay): should we buffer stderr and stdout separately?
-    p = subprocess.Popen(proc, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-
+    if len(procs) == 1:
+      processes.append(subprocess.Popen(procs[0],
+                                        stdout=subprocess.PIPE,
+                                        stderr=subprocess.STDOUT))
+    else:
+      first = True
+      for proc in procs:
+        if first:
+          processes.append(subprocess.Popen(proc,
+                                            stdout=subprocess.PIPE,
+                                            stderr=subprocess.STDOUT))
+          first = False
+        else:
+          processes.append(subprocess.Popen(proc,
+                                            stdin=processes[-1].stdout,
+                                            stdout=subprocess.PIPE,
+                                            stderr=subprocess.STDOUT))
   logging.info("started subprocess")
 
   # How long to wait (in seconds) before printing progress log messages.
@@ -68,12 +89,12 @@
   did_timeout = False
   if timeout > 0:
     wait_until = time.time() + timeout
-  while p.poll() is None and not did_timeout:
+  while processes[-1].poll() is None and not did_timeout:
     if not detach:
-      line = p.stdout.readline()
+      line = processes[-1].stdout.readline()
       while line and not did_timeout:
         _print_line(line)
-        line = p.stdout.readline()
+        line = processes[-1].stdout.readline()
         if timeout > 0:

[M-A Ruel, 2011/09/16 14:57:48]

You should seriously consider using subprocess2, it already has the timeout
logic unit tested on all platforms.

I'm also considering implementing tee-like support, I just had a few issues on
windows I had to cleanup first..

           did_timeout = time.time() > wait_until
     else:
@@ -86,7 +107,7 @@
         # TODO(erikkay): I'd prefer a less obtrusive 'print ".",' with a flush
         # but because of how we're doing subprocesses, this doesn't appear to
         # work reliably.
-        logging.info("%s still running..." % os.path.basename(proc[0]))
+        logging.info("%s still running..." % os.path.basename(procs[-1][0]))
         progress_delay_time = time.time() + progress_delay
     if timeout > 0:
       did_timeout = time.time() > wait_until
@@ -97,31 +118,47 @@
     logging.info("process ended, did not time out")
 
   if did_timeout:
-    if IsWindows():
-      subprocess.call(["taskkill", "/T", "/F", "/PID", str(p.pid)])
-    else:
-      # Does this kill all children, too?
-      os.kill(p.pid, signal.SIGINT)
-    logging.error("KILLED %d" % p.pid)
-    # Give the process a chance to actually die before continuing
-    # so that cleanup can happen safely.
-    time.sleep(1.0)
-    logging.error("TIMEOUT waiting for %s" % proc[0])
+    for p, proc in zip(processes, procs):
+      if IsWindows():
+        subprocess.call(["taskkill", "/T", "/F", "/PID", str(p.pid)])
+      else:
+        # Does this kill all children, too?
+        os.kill(p.pid, signal.SIGINT)
+      logging.error("KILLED %d" % p.pid)
+      # Give the process a chance to actually die before continuing
+      # so that cleanup can happen safely.
+      time.sleep(1.0)
+      logging.error("TIMEOUT waiting for %s" % proc[0])
     raise TimeoutError(proc[0])
   elif not detach:
-    for line in p.stdout.readlines():
+    for line in processes[-1].stdout.readlines():
       _print_line(line, False)
     if not IsMac():   # stdout flush fails on Mac
       logging.info("flushing stdout")
-      p.stdout.flush()
+      processes[-1].stdout.flush()
 
   logging.info("collecting result code")
-  result = p.poll()
+  result = processes[-1].poll()
   if result:
-    logging.error("%s exited with non-zero result code %d" % (proc[0], result))
+    logging.error("%s exited with non-zero result code %d"
+                  % (procs[0][0], result))
   return result
 
+def RunSubprocess(proc, timeout=0, detach=False, background=False):
+  """ Runs a subprocess, until it finishes or |timeout| is exceeded and the
+  process is killed with taskkill.  A |timeout| <= 0  means no timeout.
 
+  Args:
+    proc: list of process components (exe + args)
+    timeout: how long to wait before killing, <= 0 means wait forever
+    detach: Whether to pass the DETACHED_PROCESS argument to CreateProcess
+        on Windows.  This is used by Purify subprocesses on buildbot which
+        seem to get confused by the parent console that buildbot sets up.
+  """
+  return RunSubprocessChain([proc], timeout, detach, background)
+
+
+
 def IsLinux():
   return sys.platform.startswith('linux')