autotest: fix race condition between cron crash_sender and test invoked

client/site_tests/logging_CrashSender/logging_CrashSender.py

 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import logging, os, re
 from autotest_lib.client.bin import site_crash_test, site_utils, test
 from autotest_lib.client.common_lib import error, utils
 
 _25_HOURS_AGO = -25 * 60 * 60
 _CRASH_SENDER_CRON_PATH = '/etc/cron.hourly/crash_sender.hourly'
@@ skipping 15 matching lines @@
                                  board_match.group(1))
         # Get hwid
         hwclass = 'unknown'
         if os.path.exists(_HWCLASS_PATH):
             hwclass = utils.read_file(_HWCLASS_PATH)
         if not ('HWClass: %s' % hwclass) in result['output']:
             raise error.TestFail('Missing hwclass %s in output' % hwclass)
 
 
     def _check_simple_minidump_send(self, report):
-        self._set_sending(True)
         result = self._call_sender_one_crash(report=report)
         if (result['report_exists'] or
             result['rate_count'] != 1 or
             not result['send_attempt'] or
             not result['send_success'] or
             result['sleep_time'] < 0 or
             result['sleep_time'] >= _SECONDS_SEND_SPREAD or
             result['report_kind'] != 'minidump' or
             result['report_payload'] != '/var/spool/crash/fake.dmp' or
             result['exec_name'] != 'fake' or
             not 'Version: my_ver' in result['output']):
             raise error.TestFail('Simple minidump send failed')
         self._check_hardware_info(result)
 
 
     def _test_sender_simple_minidump(self):
         """Test sending a single minidump crash report."""
         self._check_simple_minidump_send(None)
 
 
     def _shift_file_mtime(self, path, delta):
         statinfo = os.stat(path)
         os.utime(path, (statinfo.st_atime,
                         statinfo.st_mtime + delta))
 
 
     def _test_sender_simple_old_minidump(self):
         """Test that old minidumps and metadata are sent."""
-        self._set_sending(True)
         dmp_path = self.write_crash_dir_entry('fake.dmp', '')
         meta_path = self.write_fake_meta('fake.meta', 'fake')
         self._shift_file_mtime(dmp_path, _25_HOURS_AGO)
         self._shift_file_mtime(meta_path, _25_HOURS_AGO)
         self._check_simple_minidump_send(meta_path)
 
 
     def _test_sender_simple_kernel_crash(self):
         """Test sending a single kcrash report."""
-        self._set_sending(True)
         kcrash_fake_report = self.write_crash_dir_entry(
             'kernel.today.kcrash', '')
         self.write_fake_meta('kernel.today.meta', 'kernel')
         result = self._call_sender_one_crash(report=kcrash_fake_report)
         if (result['report_exists'] or
             result['rate_count'] != 1 or
             not result['send_attempt'] or
             not result['send_success'] or
             result['sleep_time'] < 0 or
             result['sleep_time'] >= _SECONDS_SEND_SPREAD or
             result['report_kind'] != 'kcrash' or
             (result['report_payload'] !=
              '/var/spool/crash/kernel.today.kcrash') or
             result['exec_name'] != 'kernel'):
             raise error.TestFail('Simple kcrash send failed')
         self._check_hardware_info(result)
 
 
     def _test_sender_pausing(self):
         """Test the sender returns immediately when the pause file is present.
 
         This is testing the sender's test functionality - if this regresses,
         other tests can become flaky because the cron-started sender may run
-        asynchronously to these tests."""
-        self._set_sending(False)
+        asynchronously to these tests.  Disable child sending as normally
+        this environment configuration allows our children to run in spite of
+        the pause file."""
+        self._set_system_sending(False)
+        self._set_child_sending(False)
         result = self._call_sender_one_crash()
         if (not result['report_exists'] or
             not 'Exiting early due to' in result['output'] or
             result['send_attempt']):
             raise error.TestFail('Sender did not pause')
 
 
     def _test_sender_reports_disabled(self):
         """Test that when reporting is disabled, we don't send."""
-        self._set_sending(True)
         result = self._call_sender_one_crash(reports_enabled=False)
         if (result['report_exists'] or
             not 'Uploading is disabled' in result['output'] or
             result['send_attempt']):
             raise error.TestFail('Sender did not handle reports disabled')
 
 
     def _test_sender_rate_limiting(self):
         """Test the sender properly rate limits and sends with delay."""
-        self._set_sending(True)
         sleep_times = []
         for i in range(1, _DAILY_RATE_LIMIT + 1):
             result = self._call_sender_one_crash()
             if not result['send_attempt'] or not result['send_success']:
                 raise error.TestFail('Crash uploader did not send on #%d' % i)
             if result['rate_count'] != i:
                 raise error.TestFail('Did not properly persist rate on #%d' % i)
             sleep_times.append(result['sleep_time'])
         logging.debug('Sleeps between sending crashes were: %s' % sleep_times)
         unique_times = {}
@@ skipping 20 matching lines @@
             result['rate_count'] != _DAILY_RATE_LIMIT):
             raise error.TestFail('Crash not sent even after 25hrs pass')
 
 
     def _test_sender_single_instance(self):
         """Test the sender fails to start when another instance is running.
 
         Here we rely on the sender not checking the other running pid
         is of the same instance.
         """
-        self._set_sending(True)
         utils.open_write_close(self._CRASH_SENDER_RUN_PATH, str(os.getpid()))
         result = self._call_sender_one_crash()
         if (not 'Already running.' in result['output'] or
             result['send_attempt'] or not result['report_exists']):
             raise error.TestFail('Allowed multiple instances to run')
         os.remove(self._CRASH_SENDER_RUN_PATH)
 
 
     def _test_sender_send_fails(self):
         """Test that when the send fails we try again later."""
-        self._set_sending(True)
         result = self._call_sender_one_crash(send_success=False)
         if not result['send_attempt'] or result['send_success']:
             raise error.TestError('Did not properly cause a send failure')
         if result['rate_count'] != 1:
             raise error.TestFail('Did not count a failed send against rate '
                                  'limiting')
         if not result['report_exists']:
             raise error.TestFail('Expected minidump to be saved for later '
                                  'sending')
 
 
     def _test_sender_orphaned_files(self):
         """Test that payload and unknown files that are old are removed."""
         core_file = self.write_crash_dir_entry('random1.core', '')
         unknown_file = self.write_crash_dir_entry('.unknown', '')
         # As new files, we expect crash_sender to leave these alone.
-        self._set_sending(True)
         results = self._call_sender_one_crash()
         if ('Removing old orphaned file' in results['output'] or
             not os.path.exists(core_file) or
             not os.path.exists(unknown_file)):
             raise error.TestFail('New orphaned files were removed')
         self._shift_file_mtime(core_file, _25_HOURS_AGO)
         self._shift_file_mtime(unknown_file, _25_HOURS_AGO)
         results = self._call_sender_one_crash()
         if (not 'Removing old orphaned file' in results['output'] or
             os.path.exists(core_file) or os.path.exists(unknown_file)):
             raise error.TestFail(
                 'Old orphaned files were not removed')
 
 
     def _test_sender_incomplete_metadata(self):
         """Test that incomplete metadata file is removed once old."""
         meta_file = self.write_crash_dir_entry('incomplete.meta', 'half=1')
         dmp_file = self.write_crash_dir_entry('incomplete.dmp', '')
         # As new files, we expect crash_sender to leave these alone.
-        self._set_sending(True)
         results = self._call_sender_one_crash()
         if ('Removing recent incomplete report' in results['output'] or
             not os.path.exists(meta_file) or
             not os.path.exists(dmp_file)):
             raise error.TestFail('New unknown files were removed')
         self._shift_file_mtime(meta_file, _25_HOURS_AGO)
         results = self._call_sender_one_crash()
         if (not 'Removing old incomplete metadata' in results['output'] or
             os.path.exists(meta_file) or os.path.exists(dmp_file)):
             raise error.TestFail(
                 'Old unknown/incomplete files were not removed')
 
 
     def _test_cron_runs(self):
         """Test sender runs successfully as part of the hourly cron job.
 
         Assuming we've run test_sender_simple which shows that a minidump
         gets removed as part of sending, we run the cron job (which is
         asynchronous) and wait for that file to be removed to just verify
         the job eventually runs the sender."""
-        self._set_sending(True)
         minidump = self._prepare_sender_one_crash(send_success=True,
                                                   reports_enabled=True,
                                                   username='root',
                                                   report=None)
         if not os.path.exists(minidump):
             raise error.TestError('minidump not created')
         utils.system(_CRASH_SENDER_CRON_PATH)
         self._log_reader.set_start_by_current()
         site_utils.poll_for_condition(
             lambda: not os.path.exists(minidump),
@@ skipping 11 matching lines @@
             'sender_simple_old_minidump',
             'sender_simple_kernel_crash',
             'sender_pausing',
             'sender_reports_disabled',
             'sender_rate_limiting',
             'sender_single_instance',
             'sender_send_fails',
             'sender_orphaned_files',
             'sender_incomplete_metadata',
             'cron_runs'])