Reland Wait cloudtail termination in goma module

scripts/slave/recipe_modules/goma/resources/cloudtail_utils.py

Index: scripts/slave/recipe_modules/goma/resources/cloudtail_utils.py
diff --git a/scripts/slave/recipe_modules/goma/resources/cloudtail_utils.py b/scripts/slave/recipe_modules/goma/resources/cloudtail_utils.py
index 79f20dc1edd7c1ee81c9d20086e9b363e61c5bb5..17fa8def359c09045d0d868dae765c08f1d41383 100644
--- a/scripts/slave/recipe_modules/goma/resources/cloudtail_utils.py
+++ b/scripts/slave/recipe_modules/goma/resources/cloudtail_utils.py
@@ -9,6 +9,7 @@ import os
 import signal
 import subprocess
 import sys
+import time
 
 from slave import goma_utils
 
@@ -24,6 +25,65 @@ def start_cloudtail(args):
   with open(args.pid_file, 'w') as f:
     f.write(str(proc.pid))
 
+
+def is_running_posix(pid):
+  """Return True if process of pid is running.
+
+  Args:
+    pid(int): pid of process which this function checks
+              whether it is running or not.
+
+  Returns:
+    bool: True if process of pid is running.
+
+  Raises:
+    OSError if something happens in os.kill(pid, 0)
+  """
+
+  try:
+    os.kill(pid, 0)
+  except OSError as e:
+    if e.errno == errno.ESRCH or e.errno == errno.EPERM:
+      return False
+    raise e
+  return True
+
+
+def wait_termination(pid):
+  """Send SIGINT to pid and wait termination of pid.
+
+  Args:
+    pid(int): pid of process which this function waits termination.
+
+  Raises:
+    OSError: is_running_posix, os.waitpid and os.kill may throw OSError.

[ukai, 2016/10/12 01:48:28]

caller should SIGKILL, or this will send SIGKILL before raising exception?

[tikuta, 2016/10/12 06:51:50]

I stopped to send SIGKILL in this function.
Caller should SIGKILL.

+  """
+
+  try:
+    os.kill(pid, signal.SIGINT)
+  except:
+    os.kill(pid, signal.SIGKILL)

[ukai, 2016/10/12 01:48:28]

which case it will try send SIGKILL?

[tikuta, 2016/10/12 06:51:50]

Done.

+    raise
+
+  if os.name == 'nt':
+    try:
+      os.waitpid(pid, 0)
+    except OSError as e:
+      if e.errno == errno.ECHILD:
+        print('ignore errno.ECHILD %s' % e)

[ukai, 2016/10/12 01:48:28]

process of pid died before os.waitpid?

[tikuta, 2016/10/12 06:51:51]

Done.

+        return
+      raise e
+  else:
+    for _ in xrange(10):
+      if not is_running_posix(pid):
+        break
+      time.sleep(1)
+
+    if is_running_posix(pid):
+      os.kill(pid, signal.SIGKILL)
+      print('killed process %d running more than 10 seconds' % pid)
+
+
 def main():
   parser = argparse.ArgumentParser(
       description='cloudtail utility for goma recipe module.')
@@ -52,7 +112,14 @@ def main():
     with open(args.killed_pid_file) as f:
       # cloudtail flushes log and terminates
       # within 5 seconds when it recieves SIGINT.
-      os.kill(int(f.read()), signal.SIGINT)
+      pid = int(f.read())
+      try:
+        wait_termination(int(f.read()))
+      except OSError as e:
+        os.kill(pid, signal.SIGKILL)
+        print('killed process %d due to OSError %s' % (pid, e))
+        raise e
+
 
 if '__main__' == __name__:
   sys.exit(main())