add debug logging for win apache issues

Tools/Scripts/webkitpy/layout_tests/servers/server_base.py

Index: Tools/Scripts/webkitpy/layout_tests/servers/server_base.py
diff --git a/Tools/Scripts/webkitpy/layout_tests/servers/server_base.py b/Tools/Scripts/webkitpy/layout_tests/servers/server_base.py
index 5b7f34121a3e97bd77be4ab41029ef3aad2ccd93..6d463cf8fa02c0614ac0f852a1d4fff23d838b32 100644
--- a/Tools/Scripts/webkitpy/layout_tests/servers/server_base.py
+++ b/Tools/Scripts/webkitpy/layout_tests/servers/server_base.py
@@ -76,6 +76,7 @@ class ServerBase(object):
         self._stderr = self._executive.PIPE
         self._process = None
         self._pid = None
+        self._error_log = None
 
     def start(self):
         """Starts the server. It is an error to start an already started server.
@@ -87,6 +88,7 @@ class ServerBase(object):
         if self._filesystem.exists(self._pid_file):
             try:
                 self._pid = int(self._filesystem.read_text_file(self._pid_file))
+                _log.debug('stale %s pid file, pid %d' % (self._name, self._pid))
                 self._stop_running_server()
             except (ValueError, UnicodeDecodeError):
                 # These could be raised if the pid file is corrupt.
@@ -102,6 +104,7 @@ class ServerBase(object):
         if self._wait_for_action(self._is_server_running_on_all_ports):
             _log.debug("%s successfully started (pid = %d)" % (self._name, self._pid))
         else:
+            self._dump_logs()
             self._stop_running_server()
             raise ServerError('Failed to start %s server' % self._name)
 
@@ -171,9 +174,13 @@ class ServerBase(object):
 
     def _check_and_kill(self):
         if self._executive.check_running_pid(self._pid):
+            _log.debug('pid %d is running, killing it' % self._pid)
             host = self._port_obj.host
             self._executive.kill_process(self._pid)
             return False
+        else:
+            _log.debug('pid %d is not running' % self._pid)
+
         return True
 
     def _remove_pid_file(self):
@@ -187,6 +194,34 @@ class ServerBase(object):
                 full_path = self._filesystem.join(folder, file)
                 self._filesystem.remove(full_path)
 
+    def _dump_logs(self):
+        _log.error('%s dumping logs' % self._name)
+        if self._process:
+            _log.error('%s returncode %s' % (self._name, str(self._process.returncode)))
+            if self._process.stderr:
+                stderr_text = self._process.stderr.read()
+                if stderr_text:
+                    _log.error('%s stderr:' % self._name)
+                    for line in stderr_text.splitlines():
+                        _log.error('  %s' % line)
+                else:
+                    _log.error('%s no stderr' % self._name)
+            else:
+                _log.error('%s no stderr handle' % self._name)
+        else:
+            _log.error('%s no process' % self._name)
+        if self._error_log:
+            error_log_text = self._filesystem.read_text_file(self._error_log)
+            if error_log_text:
+                _log.error('%s error log (%s) contents:' % (self._name, self._error_log))
+                for line in error_log_text.splitlines():
+                    _log.error('  %s' % line)
+            else:
+                _log.error('%s error log empty' % self._name)
+            _log.error('')
+        else:
+            _log.error('%s no error log' % self._name)
+
     def _wait_for_action(self, action, wait_secs=20.0, sleep_secs=1.0):
         """Repeat the action for wait_sec or until it succeeds, sleeping for sleep_secs
         in between each attempt. Returns whether it succeeded."""
@@ -202,8 +237,9 @@ class ServerBase(object):
     def _is_server_running_on_all_ports(self):
         """Returns whether the server is running on all the desired ports."""
         if not self._executive.check_running_pid(self._pid):
-            _log.debug("Server isn't running at all")
-            raise ServerError("Server exited")
+            _log.error("Server isn't running at all")
+            self._dump_logs()
+            raise ServerError
 
         for mapping in self._mappings:
             s = socket.socket()
@@ -223,7 +259,7 @@ class ServerBase(object):
     def _check_that_all_ports_are_available(self):
         for mapping in self._mappings:
             s = socket.socket()
-            s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+            # s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

[Dirk Pranke, 2014/07/23 23:21:11]

I think if we set SO_REUSEADDR, on windows this was causing python to bind to a
socket that was already in use (which is bad), and so this check was not working
properly. 

Then, when we started apache, it would bail out almost immediately but because
we were throwing away the stderr output, we weren't seeing anything.

My current hypothesis is that some other process is bound to one of the ports,
possibly a python process like the buildbot slave.

             port = mapping['port']
             try:
                 s.bind(('localhost', port))
@@ -236,3 +272,4 @@ class ServerBase(object):
                     raise
             finally:
                 s.close()
+        _log.debug('all ports are available')