Kill builds that produce too much output

third_party/buildbot_slave_8_4/buildslave/runprocess.py

 # This file is part of Buildbot.  Buildbot is free software: you can
 # redistribute it and/or modify it under the terms of the GNU General Public
 # License as published by the Free Software Foundation, version 2.
 #
 # This program is distributed in the hope that it will be useful, but WITHOUT
 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
 # details.
 #
 # You should have received a copy of the GNU General Public License along with
@@ skipping 194 matching lines @@
 class RunProcess:
     """
     This is a helper class, used by slave commands to run programs in a child
     shell.
     """
 
     notreally = False
     BACKUP_TIMEOUT = 5
     KILL = "KILL"
     CHUNK_LIMIT = 128*1024
+    OUTPUT_LIMIT = 50*1024*1024

[cmp, 2014/06/10 15:51:22]

Add an inline comment ("Limit is 50 MB.")

Did you pick 50 MB based on reviewing the output in the tryserver directory?  If
you land this and any steps already produce that much output, we will have a bad
day.

And if no steps produce more than 20 MB today, let's lower this to 20 MB.  Or to
whatever amount may happen to be less than 50 MB.

 
     # Don't send any data until at least BUFFER_SIZE bytes have been collected
     # or BUFFER_TIMEOUT elapsed
     BUFFER_SIZE = 64*1024
     BUFFER_TIMEOUT = 5
 
     # For sending elapsed time:
     startTime = None
     elapsedTime = None
 
@@ skipping 90 matching lines @@
         self.timeout = timeout
         self.timer = None
         self.maxTime = maxTime
         self.maxTimer = None
         self.keepStdout = keepStdout
         self.keepStderr = keepStderr
 
         self.buffered = deque()
         self.buflen = 0
         self.buftimer = None
+        self.totalOutputLength = 0
 
         if usePTY == "slave-config":
             self.usePTY = self.builder.usePTY
         else:
             self.usePTY = usePTY
 
         # usePTY=True is a convenience for cleaning up all children and
         # grandchildren of a hung command. Fall back to usePTY=False on systems
         # and in situations where ptys cause problems.  PTYs are posix-only,
         # and for .closeStdin to matter, we must use a pipe, not a PTY
@@ skipping 279 matching lines @@
 
     def _addToBuffers(self, logname, data):
         """
         Add data to the buffer for logname
         Start a timer to send the buffers if BUFFER_TIMEOUT elapses.
         If adding data causes the buffer size to grow beyond BUFFER_SIZE, then
         the buffers will be sent.
         """
         n = len(data)
 
+        self.totalOutputLength += n
+        if self.totalOutputLength > self.OUTPUT_LIMIT:
+            self.kill('output limit (%d) exceeded' % self.OUTPUT_LIMIT)
+            return
+
         self.buflen += n
         self.buffered.append((logname, data))
         if self.buflen > self.BUFFER_SIZE:
             self._sendBuffers()
         elif not self.buftimer:
             self.buftimer = self._reactor.callLater(self.BUFFER_TIMEOUT, self._bufferTimeout)
 
     def addStdout(self, data):
         if self.sendStdout:
             self._addToBuffers('stdout', data)
@@ skipping 183 matching lines @@
         self.timer = None
         self.sendStatus({'header': "SIGKILL failed to kill process\n"})
         if self.sendRC:
             self.sendStatus({'header': "using fake rc=-1\n"})
             self.sendStatus({'rc': -1})
         slave.reboot_tools.Reboot()
         # In production, Reboot() does not return, and failed() is
         # never called. In testing mode, Reboot() returns immediately
         # with no effect, and we need to recover.
         self.failed(RuntimeError("SIGKILL failed to kill process"))