Implement RunSubprocessChain to allow the tools call several processes connected by pipes.

tools/valgrind/common.py

 # Copyright (c) 2009 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import logging
 import os
 import signal
 import subprocess
 import sys
 import time
@@ skipping 15 matching lines @@
   if flush:
     sys.stdout.flush()
 
 
 def RunSubprocessInBackground(proc):
   """Runs a subprocess in the background. Returns a handle to the process."""
   logging.info("running %s in the background" % " ".join(proc))
   return subprocess.Popen(proc)
 
 
-def RunSubprocess(proc, timeout=0, detach=False, background=False):
-  """ Runs a subprocess, until it finishes or |timeout| is exceeded and the
-  process is killed with taskkill.  A |timeout| <= 0  means no timeout.
+def RunSubprocessChain(procs, timeout=0, detach=False, background=False):
+  """ Runs a chain of subprocesses, connected by a pipe until it finishes
+  or |timeout| is exceeded and the processes are killed with taskkill.
+  A |timeout| <= 0  means no timeout.
 
   Args:
-    proc: list of process components (exe + args)
+    procs: list of commands to execute (each is a list of exe + args).
     timeout: how long to wait before killing, <= 0 means wait forever
     detach: Whether to pass the DETACHED_PROCESS argument to CreateProcess
         on Windows.  This is used by Purify subprocesses on buildbot which
         seem to get confused by the parent console that buildbot sets up.
   """
-
-  logging.info("running %s, timeout %d sec" % (" ".join(proc), timeout))
+  command_str = " | ".join(map(lambda l: " ".join(l), procs))
+  logging.info("running %s, timeout %d sec" % (command_str, timeout))
+  processes = []
   if detach:
+    if len(procs) > 1:
+      raise NotImplementedError("Chaining of detachable processes "
+                                "is not supported yet.")
     # see MSDN docs for "Process Creation Flags"
     DETACHED_PROCESS = 0x8
-    p = subprocess.Popen(proc, creationflags=DETACHED_PROCESS)
+    processes.append(subprocess.Popen(procs[0],
+                                      creationflags=DETACHED_PROCESS))
   else:
     # For non-detached processes, manually read and print out stdout and stderr.
     # By default, the subprocess is supposed to inherit these from its parent,
     # however when run under buildbot, it seems unable to read data from a
     # grandchild process, so we have to read the child and print the data as if
     # it came from us for buildbot to read it.  We're not sure why this is
     # necessary.
     # TODO(erikkay): should we buffer stderr and stdout separately?
-    p = subprocess.Popen(proc, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-
+    if len(procs) == 1:
+      processes.append(subprocess.Popen(procs[0],
+                                        stdout=subprocess.PIPE,
+                                        stderr=subprocess.STDOUT))
+    else:
+      first = True
+      for proc in procs:
+        if first:
+          processes.append(subprocess.Popen(proc,
+                                            stdout=subprocess.PIPE,
+                                            stderr=subprocess.STDOUT))
+          first = False
+        else:
+          processes.append(subprocess.Popen(proc,
+                                            stdin=processes[-1].stdout,
+                                            stdout=subprocess.PIPE,
+                                            stderr=subprocess.STDOUT))
   logging.info("started subprocess")
 
   # How long to wait (in seconds) before printing progress log messages.
   progress_delay = 300
   progress_delay_time = time.time() + progress_delay
   did_timeout = False
   if timeout > 0:
     wait_until = time.time() + timeout
-  while p.poll() is None and not did_timeout:
+  while processes[-1].poll() is None and not did_timeout:
     if not detach:
-      line = p.stdout.readline()
+      line = processes[-1].stdout.readline()
       while line and not did_timeout:
         _print_line(line)
-        line = p.stdout.readline()
+        line = processes[-1].stdout.readline()
         if timeout > 0:

[M-A Ruel, 2011/09/16 14:57:48]

You should seriously consider using subprocess2, it already has the timeout
logic unit tested on all platforms.

I'm also considering implementing tee-like support, I just had a few issues on
windows I had to cleanup first..

           did_timeout = time.time() > wait_until
     else:
       # When we detach, blocking on reading stdout doesn't work, so we sleep
       # a short time and poll.
       time.sleep(0.5)
       if time.time() >= progress_delay_time:
         # Force output on a periodic basis to avoid getting killed off by the
         # buildbot.
         # TODO(erikkay): I'd prefer a less obtrusive 'print ".",' with a flush
         # but because of how we're doing subprocesses, this doesn't appear to
         # work reliably.
-        logging.info("%s still running..." % os.path.basename(proc[0]))
+        logging.info("%s still running..." % os.path.basename(procs[-1][0]))
         progress_delay_time = time.time() + progress_delay
     if timeout > 0:
       did_timeout = time.time() > wait_until
 
   if did_timeout:
     logging.info("process timed out")
   else:
     logging.info("process ended, did not time out")
 
   if did_timeout:
-    if IsWindows():
-      subprocess.call(["taskkill", "/T", "/F", "/PID", str(p.pid)])
-    else:
-      # Does this kill all children, too?
-      os.kill(p.pid, signal.SIGINT)
-    logging.error("KILLED %d" % p.pid)
-    # Give the process a chance to actually die before continuing
-    # so that cleanup can happen safely.
-    time.sleep(1.0)
-    logging.error("TIMEOUT waiting for %s" % proc[0])
+    for p, proc in zip(processes, procs):
+      if IsWindows():
+        subprocess.call(["taskkill", "/T", "/F", "/PID", str(p.pid)])
+      else:
+        # Does this kill all children, too?
+        os.kill(p.pid, signal.SIGINT)
+      logging.error("KILLED %d" % p.pid)
+      # Give the process a chance to actually die before continuing
+      # so that cleanup can happen safely.
+      time.sleep(1.0)
+      logging.error("TIMEOUT waiting for %s" % proc[0])
     raise TimeoutError(proc[0])
   elif not detach:
-    for line in p.stdout.readlines():
+    for line in processes[-1].stdout.readlines():
       _print_line(line, False)
     if not IsMac():   # stdout flush fails on Mac
       logging.info("flushing stdout")
-      p.stdout.flush()
+      processes[-1].stdout.flush()
 
   logging.info("collecting result code")
-  result = p.poll()
+  result = processes[-1].poll()
   if result:
-    logging.error("%s exited with non-zero result code %d" % (proc[0], result))
+    logging.error("%s exited with non-zero result code %d"
+                  % (procs[0][0], result))
   return result
 
+def RunSubprocess(proc, timeout=0, detach=False, background=False):
+  """ Runs a subprocess, until it finishes or |timeout| is exceeded and the
+  process is killed with taskkill.  A |timeout| <= 0  means no timeout.
+
+  Args:
+    proc: list of process components (exe + args)
+    timeout: how long to wait before killing, <= 0 means wait forever
+    detach: Whether to pass the DETACHED_PROCESS argument to CreateProcess
+        on Windows.  This is used by Purify subprocesses on buildbot which
+        seem to get confused by the parent console that buildbot sets up.
+  """
+  return RunSubprocessChain([proc], timeout, detach, background)
+
+
 
 def IsLinux():
   return sys.platform.startswith('linux')
 
 
 def IsMac():
   return sys.platform.startswith('darwin')
 
 
 def IsWindows():
   return sys.platform == 'cygwin' or sys.platform.startswith('win')
 
 
 def PlatformNames():
   """Return an array of string to be used in paths for the platform
   (e.g. suppressions, gtest filters, ignore files etc.)
   The first element of the array describes the 'main' platform
   """
   if IsLinux():
     return ['linux']
   if IsMac():
     return ['mac']
   if IsWindows():
     return ['win32']
   raise NotImplementedError('Unknown platform "%s".' % sys.platform)