Have service_manager start a cloudtail attached to each service it starts.

infra/services/service_manager/service.py

 # Copyright (c) 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import errno
 import json
 import logging
 import os.path
 import platform
 import signal
@@ skipping 163 matching lines @@
   """Controls a running service process.
 
   Starts and stops the process, checks whether it is running, and creates and
   deletes its state file in the state_directory.
 
   State files are like UNIX PID files, except they also contain the starttime
   (read from /proc/$PID/stat) of the process to protect against the race
   condition of a different process reusing the same PID.
   """
 
-  def __init__(self, state_directory, service_config, time_fn=time.time,
-               sleep_fn=time.sleep):
+  def __init__(self, state_directory, service_config, cloudtail_path,
+               time_fn=time.time, sleep_fn=time.sleep):
     """
     Args:
       state_directory: A file will be created in this directory (with the same
           name as the service) when it is running containing its PID and
           starttime.
       service_config: A dictionary containing the service's config.  See README
           for a description of the fields.
+      cloudtail_path: Path to the cloudtail binary to use for logging, or None
+          if logging is disabled.
     """
 
     self.config = service_config
     self.name = service_config['name']
     self.root_directory = service_config['root_directory']
     self.tool = service_config['tool']
 
     self.args = service_config.get('args', [])
     self.stop_time = int(service_config.get('stop_time', 10))
 
+    if cloudtail_path is None:
+      self.cloudtail_args = None
+    else:
+      self.cloudtail_args = [
+          cloudtail_path, 'pipe',
+          '--log-id', self.name,

[Vadim Sh., 2015/09/23 17:40:21]

they have limit on number of these, as far as I understand. Since we have <2048
services, we should be fine :)

[dsansome, 2015/09/24 04:34:13]

Yep yep

+          '--local-log-level', 'info',
+      ]
+
     self._state_file = os.path.join(state_directory, self.name)
     self._time_fn = time_fn
     self._sleep_fn = sleep_fn
 
     self._process_creator = ProcessCreator.for_platform(self)
 
   def get_running_process_state(self):
     """Returns a ProcessState object about about the process.
 
     Raises some subclass of ProcessStateError if the process is not running.
@@ skipping 119 matching lines @@
 
   Makes the get_running_process_state() functionality available for the
   service_manager process.
   """
 
   def __init__(self, state_directory, root_directory, **kwargs):
     super(OwnService, self).__init__(state_directory, {
         'name': 'service_manager',
         'tool': '',
         'root_directory': root_directory,
-    }, **kwargs)
+    }, None, '', **kwargs)
     self._state_directory = state_directory
 
   def start(self):
     """Writes a state file, returns False if we're already running."""
 
     # Use a flock here to avoid a race condition where two service_managers
     # start at the same time and both write their own state file.
     try:
       with daemon.flock('service_manager.flock', self._state_directory):
         try:
@@ skipping 12 matching lines @@
     raise NotImplementedError
 
 
 class UnixProcessCreator(ProcessCreator):  # pragma: no cover
   """Implementation of ProcessCreator for Unix systems.
 
   Forks twice and sends the PID of the service over a pipe to the parent.
   """
 
   def start(self):
-    pipe_r, pipe_w = os.pipe()
+    control_r, control_w = os.pipe()
     child_pid = os.fork()
     if child_pid == 0:
-      os.close(pipe_r)
+      os.close(control_r)
       try:
-        self._start_child(os.fdopen(pipe_w, 'w'))
+        self._start_child(os.fdopen(control_w, 'w'))
       finally:
         os._exit(1)
     else:
-      os.close(pipe_w)
-      return self._start_parent(os.fdopen(pipe_r, 'r'), child_pid)
+      os.close(control_w)
+      return self._start_parent(os.fdopen(control_r, 'r'), child_pid)
 
-  def _start_child(self, pipe):
+  def _start_child(self, control_fh):
     """The part of start() that runs in the child process.
 
     Daemonises the current process, writes the new PID to the pipe, and execs
     the service executable.
     """
 
-    # Detach from the parent and close all FDs except that of the pipe.
-    daemon.become_daemon(keep_fds={pipe.fileno()})
+    # Detach from the parent but keep FDs open so we can write to the log still.
+    daemon.become_daemon(keep_fds=True)
 
     # Write our new PID to the pipe and close it.
-    json.dump({'pid': os.getpid()}, pipe)
-    pipe.close()
+    json.dump({'pid': os.getpid()}, control_fh)
+    control_fh.close()
+
+    # Start cloudtail.  This needs to be done after become_daemon so it's a
+    # child of the service itself, not service_manager.
+    # Cloudtail will inherit service_manager's stdout and stderr, so its errors
+    # will go to service_manager's log.
+    if self.service.cloudtail_args is not None:
+      log_r, log_w = os.pipe()
+      try:
+        handle = subprocess.Popen(
+            self.service.cloudtail_args, stdin=log_r, close_fds=True)
+      except OSError:
+        log_w = None
+        logging.exception('Failed to start cloudtail with args %s',
+                          self.service.cloudtail_args)
+      else:
+        logging.info('Started cloudtail for %s with PID %d',
+                     self.service.name, handle.pid)
+    else:
+      log_w = None
+
+    # Pipe our stdout and stderr to cloudtail if configured.  Close all other
+    # FDs.
+    if log_w is not None:
+      os.dup2(log_w, 1)
+      os.dup2(log_w, 2)
+      daemon.close_all_fds(keep_fds={1, 2})
+    else:
+      daemon.close_all_fds(keep_fds=None)
 
     # Exec the service.
     runpy = os.path.join(self.service.root_directory, 'run.py')
     os.execv(runpy, [runpy, self.service.tool] + self.service.args)
 
   def _start_parent(self, pipe, child_pid):
     """The part of start() that runs in the parent process.
 
     Waits for the child to start and writes a state file for the process.
     """
@@ skipping 38 matching lines @@
           stdout=null_fh,
           args=[
               os.path.join(self.service.root_directory,
                            'ENV/Scripts/python.exe'),
               os.path.join(self.service.root_directory, 'run.py'),
               self.service.tool,
           ] + self.service.args,
       )
 
     return handle.pid