Merge remote branch 'cros/upstream' into master

client/bin/base_sysinfo.py

 import os, shutil, re, glob, subprocess, logging
 
 from autotest_lib.client.common_lib import log
 from autotest_lib.client.bin import utils, package
 
 
 _DEFAULT_COMMANDS_TO_LOG_PER_TEST = []
 _DEFAULT_COMMANDS_TO_LOG_PER_BOOT = [
     "lspci -vvn", "mount",  "uptime",
     ]
@@ skipping 60 matching lines @@
     def __hash__(self):
         return hash((self.path, self.logf))
 
 
     def run(self, logdir):
         if os.path.exists(self.path):
             shutil.copyfile(self.path, os.path.join(logdir, self.logf))
 
 
 class command(loggable):
-    def __init__(self, cmd, logf=None, log_in_keyval=False):
+    def __init__(self, cmd, logf=None, log_in_keyval=False, compress_log=False):
         if not logf:
             logf = cmd.replace(" ", "_")
         super(command, self).__init__(logf, log_in_keyval)
         self.cmd = cmd
+        self._compress_log = compress_log
 
 
     def __repr__(self):
         r = "sysinfo.command(%r, %r, %r)"
         r %= (self.cmd, self.logf, self.log_in_keyval)
         return r
 
 
     def __eq__(self, other):
         if isinstance(other, command):
             return (self.cmd, self.logf) == (other.cmd, other.logf)
         elif isinstance(other, loggable):
             return False
         return NotImplemented
 
 
     def __ne__(self, other):
         result = self.__eq__(other)
         if result is NotImplemented:
             return result
         return not result
 
 
     def __hash__(self):
         return hash((self.cmd, self.logf))
 
 
     def run(self, logdir):
-        stdin = open(os.devnull, "r")
-        stdout = open(os.path.join(logdir, self.logf), "w")
-        stderr = open(os.devnull, "w")
         env = os.environ.copy()
         if "PATH" not in env:
             env["PATH"] = "/usr/bin:/bin"
-        subprocess.call(self.cmd, stdin=stdin, stdout=stdout, stderr=stderr,
-                        shell=True, env=env)
-        for f in (stdin, stdout, stderr):
-            f.close()
+        logf_path = os.path.join(logdir, self.logf)
+        stdin = open(os.devnull, "r")
+        stderr = open(os.devnull, "w")
+        stdout = open(logf_path, "w")
+        try:
+            subprocess.call(self.cmd, stdin=stdin, stdout=stdout, stderr=stderr,
+                            shell=True, env=env)
+        finally:
+            for f in (stdin, stdout, stderr):
+                f.close()
+            if self._compress_log and os.path.exists(logf_path):
+                utils.system('gzip -9 "%s"' % logf_path, ignore_status=True)
 
 
 class base_sysinfo(object):
     def __init__(self, job_resultsdir):
         self.sysinfodir = self._get_sysinfodir(job_resultsdir)
 
         # pull in the post-test logs to collect
         self.test_loggables = set()
         for cmd in _DEFAULT_COMMANDS_TO_LOG_PER_TEST:
             self.test_loggables.add(command(cmd))
@@ skipping 20 matching lines @@
         self.after_iteration_loggables = set()
         for cmd in _DEFAULT_COMMANDS_TO_LOG_AFTER_ITERATION:
             self.after_iteration_loggables.add(
                 command(cmd, logf=cmd.replace(" ", "_") + '.after'))
         for fname in _DEFAULT_FILES_TO_LOG_AFTER_ITERATION:
             self.after_iteration_loggables.add(
                 logfile(fname, logf=os.path.basename(fname) + '.after'))
 
         # add in a couple of extra files and commands we want to grab
         self.test_loggables.add(command("df -mP", logf="df"))
-        self.test_loggables.add(command("dmesg -c", logf="dmesg"))
+        # We compress the dmesg because it can get large when kernels are
+        # configured with a large buffer and some tests trigger OOMs or
+        # other large "spam" that fill it up...
+        self.test_loggables.add(command("dmesg -c", logf="dmesg",
+                                        compress_log=True))
         self.boot_loggables.add(logfile("/proc/cmdline",
                                              log_in_keyval=True))
         # log /proc/mounts but with custom filename since we already
         # log the output of the "mount" command as the filename "mount"
         self.boot_loggables.add(logfile('/proc/mounts', logf='proc_mounts'))
         self.boot_loggables.add(command("uname -a", logf="uname",
                                              log_in_keyval=True))
 
 
     def serialize(self):
@@ skipping 181 matching lines @@
             match = re.search(r"^MemTotal:\s+(\d+) kB$", mem_data,
                               re.MULTILINE)
             if match:
                 keyval["sysinfo-memtotal-in-kb"] = match.group(1)
 
         # guess the system's total physical memory, including sys tables
         keyval["sysinfo-phys-mbytes"] = utils.rounded_memtotal()//1024
 
         # return what we collected
         return keyval