autotest: Make crash tests resilient to builds with high ambient crash rates

client/site_tests/logging_UserCrash/logging_UserCrash.py

 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import grp, logging, os, pwd, re, stat, subprocess
 from signal import SIGSEGV
 from autotest_lib.client.bin import site_crash_test, site_utils, test
 from autotest_lib.client.common_lib import error, utils
 
 _COLLECTION_ERROR_SIGNATURE = 'crash_reporter-user-collection'
 _CORE2MD_PATH = '/usr/bin/core2md'
-_CORE_PATTERN = '/proc/sys/kernel/core_pattern'
 _LEAVE_CORE_PATH = '/root/.leave_core'
 _MAX_CRASH_DIRECTORY_SIZE = 32
 
 
 class logging_UserCrash(site_crash_test.CrashTest):
     version = 1
 
 
     def setup(self):
         os.chdir(self.srcdir)
         utils.make('clean all')
 
 
     def _test_reporter_startup(self):
         """Test that the core_pattern is set up by crash reporter."""
-        output = utils.read_file(_CORE_PATTERN).rstrip()
+        # Turn off crash filtering so we see the original setting.
+        self.disable_crash_filtering()
+        output = utils.read_file(self._CORE_PATTERN).rstrip()
         expected_core_pattern = ('|%s --signal=%%s --pid=%%p' %
                                  self._CRASH_REPORTER_PATH)
         if output != expected_core_pattern:
             raise error.TestFail('core pattern should have been %s, not %s' %
                                  (expected_core_pattern, output))
 
         self._log_reader.set_start_by_reboot(-1)
 
         if not self._log_reader.can_find('Enabling user crash handling'):
             raise error.TestFail(
                 'user space crash handling was not started during last boot')
 
 
     def _test_reporter_shutdown(self):
         """Test the crash_reporter shutdown code works."""
         self._log_reader.set_start_by_current()
         utils.system('%s --clean_shutdown' % self._CRASH_REPORTER_PATH)
-        output = utils.read_file(_CORE_PATTERN).rstrip()
+        output = utils.read_file(self._CORE_PATTERN).rstrip()
         if output != 'core':
             raise error.TestFail('core pattern should have been core, not %s' %
                                  output)
 
 
     def _prepare_crasher(self):
         """Extract the crasher and set its permissions.
 
         crasher is only gzipped to subvert Portage stripping.
         """
+        self.enable_crash_filtering('crasher_nobreakpad')
         self._crasher_path = os.path.join(self.srcdir, 'crasher_nobreakpad')
         utils.system('cd %s; tar xzf crasher.tgz-unmasked' %
                      self.srcdir)
 
 
     def _populate_symbols(self):
         """Set up Breakpad's symbol structure.
 
         Breakpad's minidump processor expects symbols to be in a directory
         hierarchy:
@@ skipping 326 matching lines @@
     def _test_root_crasher(self):
         """Test a user space crash when running as root is handled."""
         self._check_crashing_process('root')
 
 
     def _test_root_crasher_no_consent(self):
         """Test that without consent no files are stored."""
         results = self._check_crashing_process('root', consent=False)
 
 
+    def _check_filter_crasher(self, should_receive):
+        self._log_reader.set_start_by_current()
+        crasher_basename = os.path.basename(self._crasher_path)
+        utils.system(self._crasher_path, ignore_status=True);
+        if should_receive:
+            to_find = 'Received crash notification for ' + crasher_basename
+        else:
+            to_find = 'Ignoring crash from ' + crasher_basename
+        site_utils.poll_for_condition(
+            lambda: self._log_reader.can_find(to_find),
+            timeout=10,
+            exception=error.TestError(
+              'Timeout waiting for: ' + to_find + ' in ' +

[sosa, 2010/10/30 06:27:59]

'Timeout waiting for: %s in %s' % (to_find, blah)

+              self._log_reader.get_logs()))
+
+
+    def _test_crash_filtering(self):
+        """Test that crash filtering (a feature needed for testing) works."""
+        self._prepare_crasher()
+        crasher_basename = os.path.basename(self._crasher_path)
+        self._log_reader.set_start_by_current()
+
+        self.enable_crash_filtering('none')
+        self._check_filter_crasher(False)
+
+        self.enable_crash_filtering('sleep')
+        self._check_filter_crasher(False)
+
+        self.disable_crash_filtering()
+        self._check_filter_crasher(True)
+
+
     def _test_max_enqueued_crashes(self):
         """Test that _MAX_CRASH_DIRECTORY_SIZE is enforced."""
         self._log_reader.set_start_by_current()
         username = 'root'
 
         crash_dir = self._get_crash_dir(username)
         full_message = ('Crash directory %s already full with %d pending '
                         'reports' % (crash_dir, _MAX_CRASH_DIRECTORY_SIZE))
 
         # Fill up the queue.
@@ skipping 24 matching lines @@
 
           if crash_dir_size != len(os.listdir(crash_dir)):
             utils.system('ls -l %s' % crash_dir)
             raise error.TestFail('expected no new files (now %d were %d)',
                                  len(os.listdir(crash_dir)),
                                  crash_dir_size)
 
 
     def _check_collection_failure(self, test_option, failure_string):
         # Add parameter to core_pattern.
-        old_core_pattern = utils.read_file(_CORE_PATTERN)[:-1]
+        old_core_pattern = utils.read_file(self._CORE_PATTERN)[:-1]
         try:
             utils.system('echo "%s %s" > %s' % (old_core_pattern, test_option,
-                                                _CORE_PATTERN))
+                                                self._CORE_PATTERN))
             result = self._run_crasher_process_and_analyze('root',
                                                            consent=True)
             self._check_crashed_and_caught(result)
             if not self._log_reader.can_find(failure_string):
                 raise error.TestFail('Did not find fail string in log %s' %
                                      failure_string)
             if result['minidump']:
                 raise error.TestFail('failed collection resulted in minidump')
             if not result['log']:
                 raise error.TestFail('failed collection had no log')
             log_contents = utils.read_file(result['log'])
             if not log_contents.startswith(failure_string):
                 raise error.TestFail('Expected logged error '
                                      '\"%s\" was \"%s\"' %
                                      (failure_string, log_contents))
             self._check_generated_report_sending(result['meta'],
                                                  result['log'],
                                                  'root',
                                                  result['basename'],
                                                  'log',
                                                  _COLLECTION_ERROR_SIGNATURE)
         finally:
-            utils.system('echo "%s" > %s' % (old_core_pattern, _CORE_PATTERN))
+            utils.system('echo "%s" > %s' % (old_core_pattern,
+                                             self._CORE_PATTERN))
 
 
     def _test_core2md_failure(self):
         self._check_collection_failure('--core2md_failure_test',
                                        'Problem during %s [result=1]: Usage:' %
                                        _CORE2MD_PATH)
 
 
     def _test_internal_directory_failure(self):
         self._check_collection_failure('--directory_failure_test',
@@ skipping 64 matching lines @@
     # non-root, non-chronos user.
 
     def run_once(self):
         self.run_crash_tests(['reporter_startup',
                               'reporter_shutdown',
                               'no_crash',
                               'chronos_crasher',
                               'chronos_crasher_no_consent',
                               'root_crasher',
                               'root_crasher_no_consent',
+                              'crash_filtering',
                               'max_enqueued_crashes',
                               'core2md_failure',
                               'internal_directory_failure',
                               'core_file_persists_in_debug',
                               'core_file_removed_in_production'],
                               initialize_crash_reporter = True)