| OLD | NEW |
| 1 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved. | 1 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 import logging, os | 5 import logging, os |
| 6 from autotest_lib.client.bin import site_crash_test, site_utils, test | 6 from autotest_lib.client.bin import site_crash_test, site_utils, test |
| 7 from autotest_lib.client.common_lib import error, utils | 7 from autotest_lib.client.common_lib import error, utils |
| 8 | 8 |
| 9 _CRASH_SENDER_CRON_PATH = '/etc/cron.hourly/crash_sender.hourly' | 9 _CRASH_SENDER_CRON_PATH = '/etc/cron.hourly/crash_sender.hourly' |
| 10 _CRASH_SENDER_RUN_PATH = '/var/run/crash_sender.pid' | 10 _CRASH_SENDER_RUN_PATH = '/var/run/crash_sender.pid' |
| 11 _DAILY_RATE_LIMIT = 8 | 11 _DAILY_RATE_LIMIT = 8 |
| 12 _MIN_UNIQUE_TIMES = 4 | 12 _MIN_UNIQUE_TIMES = 4 |
| 13 _SECONDS_SEND_SPREAD = 3600 | 13 _SECONDS_SEND_SPREAD = 3600 |
| 14 | 14 |
| 15 | 15 |
| 16 class logging_CrashSender(site_crash_test.CrashTest): | 16 class logging_CrashSender(site_crash_test.CrashTest): |
| 17 version = 1 | 17 version = 1 |
| 18 | 18 |
| 19 | 19 |
| 20 def _test_sender_simple(self): | 20 def _test_sender_simple_minidump(self): |
| 21 """Test sending a single crash.""" | 21 """Test sending a single minidump crash report.""" |
| 22 self._set_sending(True) | 22 self._set_sending(True) |
| 23 result = self._call_sender_one_crash() | 23 result = self._call_sender_one_crash() |
| 24 if (result['minidump_exists'] or | 24 if (result['report_exists'] or |
| 25 result['rate_count'] != 1 or | 25 result['rate_count'] != 1 or |
| 26 not result['send_attempt'] or | 26 not result['send_attempt'] or |
| 27 not result['send_success'] or | 27 not result['send_success'] or |
| 28 result['sleep_time'] < 0 or | 28 result['sleep_time'] < 0 or |
| 29 result['sleep_time'] >= _SECONDS_SEND_SPREAD): | 29 result['sleep_time'] >= _SECONDS_SEND_SPREAD or |
| 30 raise error.TestFail('Simple send failed') | 30 result['report_kind'] != 'minidump' or |
| 31 result['exec_name'] != 'fake'): |
| 32 raise error.TestFail('Simple minidump send failed') |
| 33 |
| 34 |
| 35 def _test_sender_simple_kernel_crash(self): |
| 36 """Test sending a single kcrash report.""" |
| 37 self._set_sending(True) |
| 38 kcrash_fake_report = self.create_fake_crash_dir_entry( |
| 39 'kernel.today.kcrash') |
| 40 result = self._call_sender_one_crash(report=kcrash_fake_report) |
| 41 if (result['report_exists'] or |
| 42 result['rate_count'] != 1 or |
| 43 not result['send_attempt'] or |
| 44 not result['send_success'] or |
| 45 result['sleep_time'] < 0 or |
| 46 result['sleep_time'] >= _SECONDS_SEND_SPREAD or |
| 47 result['report_kind'] != 'kcrash' or |
| 48 result['exec_name'] != 'kernel'): |
| 49 raise error.TestFail('Simple kcrash send failed') |
| 31 | 50 |
| 32 | 51 |
| 33 def _test_sender_pausing(self): | 52 def _test_sender_pausing(self): |
| 34 """Test the sender returns immediately when the pause file is present. | 53 """Test the sender returns immediately when the pause file is present. |
| 35 | 54 |
| 36 This is testing the sender's test functionality - if this regresses, | 55 This is testing the sender's test functionality - if this regresses, |
| 37 other tests can become flaky because the cron-started sender may run | 56 other tests can become flaky because the cron-started sender may run |
| 38 asynchronously to these tests.""" | 57 asynchronously to these tests.""" |
| 39 self._set_sending(False) | 58 self._set_sending(False) |
| 40 result = self._call_sender_one_crash() | 59 result = self._call_sender_one_crash() |
| 41 if (not result['minidump_exists'] or | 60 if (not result['report_exists'] or |
| 42 not 'Exiting early due to' in result['output'] or | 61 not 'Exiting early due to' in result['output'] or |
| 43 result['send_attempt']): | 62 result['send_attempt']): |
| 44 raise error.TestFail('Sender did not pause') | 63 raise error.TestFail('Sender did not pause') |
| 45 | 64 |
| 46 | 65 |
| 47 def _test_sender_reports_disabled(self): | 66 def _test_sender_reports_disabled(self): |
| 48 """Test that when reporting is disabled, we don't send.""" | 67 """Test that when reporting is disabled, we don't send.""" |
| 49 self._set_sending(True) | 68 self._set_sending(True) |
| 50 result = self._call_sender_one_crash(reports_enabled=False) | 69 result = self._call_sender_one_crash(reports_enabled=False) |
| 51 if (result['minidump_exists'] or | 70 if (result['report_exists'] or |
| 52 not 'Uploading is disabled' in result['output'] or | 71 not 'Uploading is disabled' in result['output'] or |
| 53 result['send_attempt']): | 72 result['send_attempt']): |
| 54 raise error.TestFail('Sender did not handle reports disabled') | 73 raise error.TestFail('Sender did not handle reports disabled') |
| 55 | 74 |
| 56 | 75 |
| 57 def _test_sender_rate_limiting(self): | 76 def _test_sender_rate_limiting(self): |
| 58 """Test the sender properly rate limits and sends with delay.""" | 77 """Test the sender properly rate limits and sends with delay.""" |
| 59 self._set_sending(True) | 78 self._set_sending(True) |
| 60 sleep_times = [] | 79 sleep_times = [] |
| 61 for i in range(1, _DAILY_RATE_LIMIT + 1): | 80 for i in range(1, _DAILY_RATE_LIMIT + 1): |
| 62 result = self._call_sender_one_crash() | 81 result = self._call_sender_one_crash() |
| 63 if not result['send_attempt'] or not result['send_success']: | 82 if not result['send_attempt'] or not result['send_success']: |
| 64 raise error.TestFail('Crash uploader did not send on #%d' % i) | 83 raise error.TestFail('Crash uploader did not send on #%d' % i) |
| 65 if result['rate_count'] != i: | 84 if result['rate_count'] != i: |
| 66 raise error.TestFail('Did not properly persist rate on #%d' % i) | 85 raise error.TestFail('Did not properly persist rate on #%d' % i) |
| 67 sleep_times.append(result['sleep_time']) | 86 sleep_times.append(result['sleep_time']) |
| 68 logging.debug('Sleeps between sending crashes were: %s' % sleep_times) | 87 logging.debug('Sleeps between sending crashes were: %s' % sleep_times) |
| 69 unique_times = {} | 88 unique_times = {} |
| 70 for i in range(0, _DAILY_RATE_LIMIT): | 89 for i in range(0, _DAILY_RATE_LIMIT): |
| 71 unique_times[sleep_times[i]] = True | 90 unique_times[sleep_times[i]] = True |
| 72 if len(unique_times) < _MIN_UNIQUE_TIMES: | 91 if len(unique_times) < _MIN_UNIQUE_TIMES: |
| 73 raise error.TestFail('Expected at least %d unique times: %s' % | 92 raise error.TestFail('Expected at least %d unique times: %s' % |
| 74 _MIN_UNIQUE_TIMES, sleep_times) | 93 _MIN_UNIQUE_TIMES, sleep_times) |
| 75 # Now the _DAILY_RATE_LIMIT ^ th send request should fail. | 94 # Now the _DAILY_RATE_LIMIT ^ th send request should fail. |
| 76 result = self._call_sender_one_crash() | 95 result = self._call_sender_one_crash() |
| 77 if (not result['minidump_exists'] or | 96 if (not result['report_exists'] or |
| 78 not 'Cannot send more crashes' in result['output'] or | 97 not 'Cannot send more crashes' in result['output'] or |
| 79 result['rate_count'] != _DAILY_RATE_LIMIT): | 98 result['rate_count'] != _DAILY_RATE_LIMIT): |
| 80 raise error.TestFail('Crash rate limiting did not take effect') | 99 raise error.TestFail('Crash rate limiting did not take effect') |
| 81 | 100 |
| 82 # Set one rate file a day earlier and verify can send | 101 # Set one rate file a day earlier and verify can send |
| 83 rate_files = os.listdir(self._CRASH_SENDER_RATE_DIR) | 102 rate_files = os.listdir(self._CRASH_SENDER_RATE_DIR) |
| 84 rate_path = os.path.join(self._CRASH_SENDER_RATE_DIR, rate_files[0]) | 103 rate_path = os.path.join(self._CRASH_SENDER_RATE_DIR, rate_files[0]) |
| 85 statinfo = os.stat(rate_path) | 104 statinfo = os.stat(rate_path) |
| 86 os.utime(rate_path, (statinfo.st_atime, | 105 os.utime(rate_path, (statinfo.st_atime, |
| 87 statinfo.st_mtime - (60 * 60 * 25))) | 106 statinfo.st_mtime - (60 * 60 * 25))) |
| 88 utils.system('ls -l ' + self._CRASH_SENDER_RATE_DIR) | 107 utils.system('ls -l ' + self._CRASH_SENDER_RATE_DIR) |
| 89 result = self._call_sender_one_crash() | 108 result = self._call_sender_one_crash() |
| 90 if (not result['send_attempt'] or | 109 if (not result['send_attempt'] or |
| 91 not result['send_success'] or | 110 not result['send_success'] or |
| 92 result['rate_count'] != _DAILY_RATE_LIMIT): | 111 result['rate_count'] != _DAILY_RATE_LIMIT): |
| 93 raise error.TestFail('Crash not sent even after 25hrs pass') | 112 raise error.TestFail('Crash not sent even after 25hrs pass') |
| 94 | 113 |
| 95 | 114 |
| 96 def _test_sender_single_instance(self): | 115 def _test_sender_single_instance(self): |
| 97 """Test the sender fails to start when another instance is running. | 116 """Test the sender fails to start when another instance is running. |
| 98 | 117 |
| 99 Here we rely on the sender not checking the other running pid | 118 Here we rely on the sender not checking the other running pid |
| 100 is of the same instance. | 119 is of the same instance. |
| 101 """ | 120 """ |
| 102 self._set_sending(True) | 121 self._set_sending(True) |
| 103 utils.open_write_close(_CRASH_SENDER_RUN_PATH, str(os.getpid())) | 122 utils.open_write_close(_CRASH_SENDER_RUN_PATH, str(os.getpid())) |
| 104 result = self._call_sender_one_crash() | 123 result = self._call_sender_one_crash() |
| 105 if (not 'Already running.' in result['output'] or | 124 if (not 'Already running.' in result['output'] or |
| 106 result['send_attempt'] or not result['minidump_exists']): | 125 result['send_attempt'] or not result['report_exists']): |
| 107 raise error.TestFail('Allowed multiple instances to run') | 126 raise error.TestFail('Allowed multiple instances to run') |
| 108 os.remove(_CRASH_SENDER_RUN_PATH) | 127 os.remove(_CRASH_SENDER_RUN_PATH) |
| 109 | 128 |
| 110 | 129 |
| 111 def _test_sender_send_fails(self): | 130 def _test_sender_send_fails(self): |
| 112 """Test that when the send fails we try again later.""" | 131 """Test that when the send fails we try again later.""" |
| 113 self._set_sending(True) | 132 self._set_sending(True) |
| 114 result = self._call_sender_one_crash(send_success=False) | 133 result = self._call_sender_one_crash(send_success=False) |
| 115 if not result['send_attempt'] or result['send_success']: | 134 if not result['send_attempt'] or result['send_success']: |
| 116 raise error.TestError('Did not properly cause a send failure') | 135 raise error.TestError('Did not properly cause a send failure') |
| 117 if result['rate_count'] != 1: | 136 if result['rate_count'] != 1: |
| 118 raise error.TestFail('Did not count a failed send against rate ' | 137 raise error.TestFail('Did not count a failed send against rate ' |
| 119 'limiting') | 138 'limiting') |
| 120 if not result['minidump_exists']: | 139 if not result['report_exists']: |
| 121 raise error.TestFail('Expected minidump to be saved for later ' | 140 raise error.TestFail('Expected minidump to be saved for later ' |
| 122 'sending') | 141 'sending') |
| 123 | 142 |
| 124 | 143 |
| 125 def _test_sender_leaves_core_files(self): | 144 def _test_sender_leaves_core_files(self): |
| 126 """Test that a core file is left in the send directory. | 145 """Test that a core file is left in the send directory. |
| 127 | 146 |
| 128 Core files will only persist for developer/testing images. We | 147 Core files will only persist for developer/testing images. We |
| 129 should never remove such a file.""" | 148 should never remove such a file.""" |
| 130 self._set_sending(True) | 149 self._set_sending(True) |
| 131 # Call prepare function to make sure the directory exists. | 150 # Call prepare function to make sure the directory exists. |
| 132 core_name = 'something.ending.with.core' | 151 core_name = 'something.ending.with.core' |
| 133 core_path = self._create_fake_crash_dir_entry(core_name) | 152 core_path = self.create_fake_crash_dir_entry(core_name) |
| 134 result = self._call_sender_one_crash() | 153 result = self._call_sender_one_crash() |
| 135 if not 'Ignoring core file.' in result['output']: | 154 if not 'Ignoring core file.' in result['output']: |
| 136 raise error.TestFail('Expected ignoring core file message') | 155 raise error.TestFail('Expected ignoring core file message') |
| 137 if not os.path.exists(core_path): | 156 if not os.path.exists(core_path): |
| 138 raise error.TestFail('Core file was removed') | 157 raise error.TestFail('Core file was removed') |
| 139 | 158 |
| 140 | 159 |
| 160 def _test_sender_unknown_report_kind(self): |
| 161 self._set_sending(True) |
| 162 bad_report = self.create_fake_crash_dir_entry('fake.bad') |
| 163 result = self._call_sender_one_crash(report=bad_report) |
| 164 if (result['report_exists'] or |
| 165 result['rate_count'] != 0 or |
| 166 result['send_attempt'] or |
| 167 result['send_success'] or |
| 168 not 'Unknown report' in result['output']): |
| 169 raise error.TestFail('Error handling of unknown report kind failed') |
| 170 |
| 171 |
| 141 def _test_cron_runs(self): | 172 def _test_cron_runs(self): |
| 142 """Test sender runs successfully as part of the hourly cron job. | 173 """Test sender runs successfully as part of the hourly cron job. |
| 143 | 174 |
| 144 Assuming we've run test_sender_simple which shows that a minidump | 175 Assuming we've run test_sender_simple which shows that a minidump |
| 145 gets removed as part of sending, we run the cron job (which is | 176 gets removed as part of sending, we run the cron job (which is |
| 146 asynchronous) and wait for that file to be removed to just verify | 177 asynchronous) and wait for that file to be removed to just verify |
| 147 the job eventually runs the sender.""" | 178 the job eventually runs the sender.""" |
| 148 self._set_sending(True) | 179 self._set_sending(True) |
| 149 minidump = self._prepare_sender_one_crash(send_success=True, | 180 minidump = self._prepare_sender_one_crash(send_success=True, |
| 150 reports_enabled=True, | 181 reports_enabled=True, |
| 151 username='root', | 182 username='root', |
| 152 minidump=None) | 183 report=None) |
| 153 if not os.path.exists(minidump): | 184 if not os.path.exists(minidump): |
| 154 raise error.TestError('minidump not created') | 185 raise error.TestError('minidump not created') |
| 155 utils.system(_CRASH_SENDER_CRON_PATH) | 186 utils.system(_CRASH_SENDER_CRON_PATH) |
| 156 self._log_reader.set_start_by_current() | 187 self._log_reader.set_start_by_current() |
| 157 site_utils.poll_for_condition( | 188 site_utils.poll_for_condition( |
| 158 lambda: not os.path.exists(minidump), | 189 lambda: not os.path.exists(minidump), |
| 159 desc='minidump to be removed') | 190 desc='minidump to be removed') |
| 160 crash_sender_log = self._log_reader.get_logs() | 191 crash_sender_log = self._log_reader.get_logs() |
| 161 logging.debug('Contents of crash sender log: ' + crash_sender_log) | 192 logging.debug('Contents of crash sender log: ' + crash_sender_log) |
| 162 result = self._parse_sender_output(crash_sender_log) | 193 result = self._parse_sender_output(crash_sender_log) |
| 163 if not result['send_attempt'] or not result['send_success']: | 194 if not result['send_attempt'] or not result['send_success']: |
| 164 raise error.TestFail('Cron simple run test failed') | 195 raise error.TestFail('Cron simple run test failed') |
| 165 | 196 |
| 166 | 197 |
| 167 def run_once(self): | 198 def run_once(self): |
| 168 self.run_crash_tests([ | 199 self.run_crash_tests([ |
| 169 'sender_simple', | 200 'sender_simple_minidump', |
| 201 'sender_simple_kernel_crash', |
| 170 'sender_pausing', | 202 'sender_pausing', |
| 171 'sender_reports_disabled', | 203 'sender_reports_disabled', |
| 172 'sender_rate_limiting', | 204 'sender_rate_limiting', |
| 173 'sender_single_instance', | 205 'sender_single_instance', |
| 174 'sender_send_fails', | 206 'sender_send_fails', |
| 175 'sender_leaves_core_files', | 207 'sender_leaves_core_files', |
| 208 'sender_unknown_report_kind', |
| 176 'cron_runs']) | 209 'cron_runs']) |
| OLD | NEW |