OLD | NEW |
1 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved. | 1 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 import logging, os | 5 import logging, os |
6 from autotest_lib.client.bin import site_crash_test, site_utils, test | 6 from autotest_lib.client.bin import site_crash_test, site_utils, test |
7 from autotest_lib.client.common_lib import error, utils | 7 from autotest_lib.client.common_lib import error, utils |
8 | 8 |
9 _CRASH_SENDER_CRON_PATH = '/etc/cron.hourly/crash_sender.hourly' | 9 _CRASH_SENDER_CRON_PATH = '/etc/cron.hourly/crash_sender.hourly' |
10 _CRASH_SENDER_RUN_PATH = '/var/run/crash_sender.pid' | 10 _CRASH_SENDER_RUN_PATH = '/var/run/crash_sender.pid' |
11 _DAILY_RATE_LIMIT = 8 | 11 _DAILY_RATE_LIMIT = 8 |
12 _MIN_UNIQUE_TIMES = 4 | 12 _MIN_UNIQUE_TIMES = 4 |
13 _SECONDS_SEND_SPREAD = 3600 | 13 _SECONDS_SEND_SPREAD = 3600 |
14 | 14 |
15 | 15 |
16 class logging_CrashSender(site_crash_test.CrashTest): | 16 class logging_CrashSender(site_crash_test.CrashTest): |
17 version = 1 | 17 version = 1 |
18 | 18 |
19 | 19 |
20 def _test_sender_simple(self): | 20 def _test_sender_simple_minidump(self): |
21 """Test sending a single crash.""" | 21 """Test sending a single minidump crash report.""" |
22 self._set_sending(True) | 22 self._set_sending(True) |
23 result = self._call_sender_one_crash() | 23 result = self._call_sender_one_crash() |
24 if (result['minidump_exists'] or | 24 if (result['report_exists'] or |
25 result['rate_count'] != 1 or | 25 result['rate_count'] != 1 or |
26 not result['send_attempt'] or | 26 not result['send_attempt'] or |
27 not result['send_success'] or | 27 not result['send_success'] or |
28 result['sleep_time'] < 0 or | 28 result['sleep_time'] < 0 or |
29 result['sleep_time'] >= _SECONDS_SEND_SPREAD): | 29 result['sleep_time'] >= _SECONDS_SEND_SPREAD or |
30 raise error.TestFail('Simple send failed') | 30 result['report_kind'] != 'minidump' or |
| 31 result['exec_name'] != 'fake'): |
| 32 raise error.TestFail('Simple minidump send failed') |
| 33 |
| 34 |
| 35 def _test_sender_simple_kernel_crash(self): |
| 36 """Test sending a single kcrash report.""" |
| 37 self._set_sending(True) |
| 38 kcrash_fake_report = self.create_fake_crash_dir_entry( |
| 39 'kernel.today.kcrash') |
| 40 result = self._call_sender_one_crash(report=kcrash_fake_report) |
| 41 if (result['report_exists'] or |
| 42 result['rate_count'] != 1 or |
| 43 not result['send_attempt'] or |
| 44 not result['send_success'] or |
| 45 result['sleep_time'] < 0 or |
| 46 result['sleep_time'] >= _SECONDS_SEND_SPREAD or |
| 47 result['report_kind'] != 'kcrash' or |
| 48 result['exec_name'] != 'kernel'): |
| 49 raise error.TestFail('Simple kcrash send failed') |
31 | 50 |
32 | 51 |
33 def _test_sender_pausing(self): | 52 def _test_sender_pausing(self): |
34 """Test the sender returns immediately when the pause file is present. | 53 """Test the sender returns immediately when the pause file is present. |
35 | 54 |
36 This is testing the sender's test functionality - if this regresses, | 55 This is testing the sender's test functionality - if this regresses, |
37 other tests can become flaky because the cron-started sender may run | 56 other tests can become flaky because the cron-started sender may run |
38 asynchronously to these tests.""" | 57 asynchronously to these tests.""" |
39 self._set_sending(False) | 58 self._set_sending(False) |
40 result = self._call_sender_one_crash() | 59 result = self._call_sender_one_crash() |
41 if (not result['minidump_exists'] or | 60 if (not result['report_exists'] or |
42 not 'Exiting early due to' in result['output'] or | 61 not 'Exiting early due to' in result['output'] or |
43 result['send_attempt']): | 62 result['send_attempt']): |
44 raise error.TestFail('Sender did not pause') | 63 raise error.TestFail('Sender did not pause') |
45 | 64 |
46 | 65 |
47 def _test_sender_reports_disabled(self): | 66 def _test_sender_reports_disabled(self): |
48 """Test that when reporting is disabled, we don't send.""" | 67 """Test that when reporting is disabled, we don't send.""" |
49 self._set_sending(True) | 68 self._set_sending(True) |
50 result = self._call_sender_one_crash(reports_enabled=False) | 69 result = self._call_sender_one_crash(reports_enabled=False) |
51 if (result['minidump_exists'] or | 70 if (result['report_exists'] or |
52 not 'Uploading is disabled' in result['output'] or | 71 not 'Uploading is disabled' in result['output'] or |
53 result['send_attempt']): | 72 result['send_attempt']): |
54 raise error.TestFail('Sender did not handle reports disabled') | 73 raise error.TestFail('Sender did not handle reports disabled') |
55 | 74 |
56 | 75 |
57 def _test_sender_rate_limiting(self): | 76 def _test_sender_rate_limiting(self): |
58 """Test the sender properly rate limits and sends with delay.""" | 77 """Test the sender properly rate limits and sends with delay.""" |
59 self._set_sending(True) | 78 self._set_sending(True) |
60 sleep_times = [] | 79 sleep_times = [] |
61 for i in range(1, _DAILY_RATE_LIMIT + 1): | 80 for i in range(1, _DAILY_RATE_LIMIT + 1): |
62 result = self._call_sender_one_crash() | 81 result = self._call_sender_one_crash() |
63 if not result['send_attempt'] or not result['send_success']: | 82 if not result['send_attempt'] or not result['send_success']: |
64 raise error.TestFail('Crash uploader did not send on #%d' % i) | 83 raise error.TestFail('Crash uploader did not send on #%d' % i) |
65 if result['rate_count'] != i: | 84 if result['rate_count'] != i: |
66 raise error.TestFail('Did not properly persist rate on #%d' % i) | 85 raise error.TestFail('Did not properly persist rate on #%d' % i) |
67 sleep_times.append(result['sleep_time']) | 86 sleep_times.append(result['sleep_time']) |
68 logging.debug('Sleeps between sending crashes were: %s' % sleep_times) | 87 logging.debug('Sleeps between sending crashes were: %s' % sleep_times) |
69 unique_times = {} | 88 unique_times = {} |
70 for i in range(0, _DAILY_RATE_LIMIT): | 89 for i in range(0, _DAILY_RATE_LIMIT): |
71 unique_times[sleep_times[i]] = True | 90 unique_times[sleep_times[i]] = True |
72 if len(unique_times) < _MIN_UNIQUE_TIMES: | 91 if len(unique_times) < _MIN_UNIQUE_TIMES: |
73 raise error.TestFail('Expected at least %d unique times: %s' % | 92 raise error.TestFail('Expected at least %d unique times: %s' % |
74 _MIN_UNIQUE_TIMES, sleep_times) | 93 _MIN_UNIQUE_TIMES, sleep_times) |
75 # Now the _DAILY_RATE_LIMIT ^ th send request should fail. | 94 # Now the _DAILY_RATE_LIMIT ^ th send request should fail. |
76 result = self._call_sender_one_crash() | 95 result = self._call_sender_one_crash() |
77 if (not result['minidump_exists'] or | 96 if (not result['report_exists'] or |
78 not 'Cannot send more crashes' in result['output'] or | 97 not 'Cannot send more crashes' in result['output'] or |
79 result['rate_count'] != _DAILY_RATE_LIMIT): | 98 result['rate_count'] != _DAILY_RATE_LIMIT): |
80 raise error.TestFail('Crash rate limiting did not take effect') | 99 raise error.TestFail('Crash rate limiting did not take effect') |
81 | 100 |
82 # Set one rate file a day earlier and verify can send | 101 # Set one rate file a day earlier and verify can send |
83 rate_files = os.listdir(self._CRASH_SENDER_RATE_DIR) | 102 rate_files = os.listdir(self._CRASH_SENDER_RATE_DIR) |
84 rate_path = os.path.join(self._CRASH_SENDER_RATE_DIR, rate_files[0]) | 103 rate_path = os.path.join(self._CRASH_SENDER_RATE_DIR, rate_files[0]) |
85 statinfo = os.stat(rate_path) | 104 statinfo = os.stat(rate_path) |
86 os.utime(rate_path, (statinfo.st_atime, | 105 os.utime(rate_path, (statinfo.st_atime, |
87 statinfo.st_mtime - (60 * 60 * 25))) | 106 statinfo.st_mtime - (60 * 60 * 25))) |
88 utils.system('ls -l ' + self._CRASH_SENDER_RATE_DIR) | 107 utils.system('ls -l ' + self._CRASH_SENDER_RATE_DIR) |
89 result = self._call_sender_one_crash() | 108 result = self._call_sender_one_crash() |
90 if (not result['send_attempt'] or | 109 if (not result['send_attempt'] or |
91 not result['send_success'] or | 110 not result['send_success'] or |
92 result['rate_count'] != _DAILY_RATE_LIMIT): | 111 result['rate_count'] != _DAILY_RATE_LIMIT): |
93 raise error.TestFail('Crash not sent even after 25hrs pass') | 112 raise error.TestFail('Crash not sent even after 25hrs pass') |
94 | 113 |
95 | 114 |
96 def _test_sender_single_instance(self): | 115 def _test_sender_single_instance(self): |
97 """Test the sender fails to start when another instance is running. | 116 """Test the sender fails to start when another instance is running. |
98 | 117 |
99 Here we rely on the sender not checking the other running pid | 118 Here we rely on the sender not checking the other running pid |
100 is of the same instance. | 119 is of the same instance. |
101 """ | 120 """ |
102 self._set_sending(True) | 121 self._set_sending(True) |
103 utils.open_write_close(_CRASH_SENDER_RUN_PATH, str(os.getpid())) | 122 utils.open_write_close(_CRASH_SENDER_RUN_PATH, str(os.getpid())) |
104 result = self._call_sender_one_crash() | 123 result = self._call_sender_one_crash() |
105 if (not 'Already running.' in result['output'] or | 124 if (not 'Already running.' in result['output'] or |
106 result['send_attempt'] or not result['minidump_exists']): | 125 result['send_attempt'] or not result['report_exists']): |
107 raise error.TestFail('Allowed multiple instances to run') | 126 raise error.TestFail('Allowed multiple instances to run') |
108 os.remove(_CRASH_SENDER_RUN_PATH) | 127 os.remove(_CRASH_SENDER_RUN_PATH) |
109 | 128 |
110 | 129 |
111 def _test_sender_send_fails(self): | 130 def _test_sender_send_fails(self): |
112 """Test that when the send fails we try again later.""" | 131 """Test that when the send fails we try again later.""" |
113 self._set_sending(True) | 132 self._set_sending(True) |
114 result = self._call_sender_one_crash(send_success=False) | 133 result = self._call_sender_one_crash(send_success=False) |
115 if not result['send_attempt'] or result['send_success']: | 134 if not result['send_attempt'] or result['send_success']: |
116 raise error.TestError('Did not properly cause a send failure') | 135 raise error.TestError('Did not properly cause a send failure') |
117 if result['rate_count'] != 1: | 136 if result['rate_count'] != 1: |
118 raise error.TestFail('Did not count a failed send against rate ' | 137 raise error.TestFail('Did not count a failed send against rate ' |
119 'limiting') | 138 'limiting') |
120 if not result['minidump_exists']: | 139 if not result['report_exists']: |
121 raise error.TestFail('Expected minidump to be saved for later ' | 140 raise error.TestFail('Expected minidump to be saved for later ' |
122 'sending') | 141 'sending') |
123 | 142 |
124 | 143 |
125 def _test_sender_leaves_core_files(self): | 144 def _test_sender_leaves_core_files(self): |
126 """Test that a core file is left in the send directory. | 145 """Test that a core file is left in the send directory. |
127 | 146 |
128 Core files will only persist for developer/testing images. We | 147 Core files will only persist for developer/testing images. We |
129 should never remove such a file.""" | 148 should never remove such a file.""" |
130 self._set_sending(True) | 149 self._set_sending(True) |
131 # Call prepare function to make sure the directory exists. | 150 # Call prepare function to make sure the directory exists. |
132 core_name = 'something.ending.with.core' | 151 core_name = 'something.ending.with.core' |
133 core_path = self._create_fake_crash_dir_entry(core_name) | 152 core_path = self.create_fake_crash_dir_entry(core_name) |
134 result = self._call_sender_one_crash() | 153 result = self._call_sender_one_crash() |
135 if not 'Ignoring core file.' in result['output']: | 154 if not 'Ignoring core file.' in result['output']: |
136 raise error.TestFail('Expected ignoring core file message') | 155 raise error.TestFail('Expected ignoring core file message') |
137 if not os.path.exists(core_path): | 156 if not os.path.exists(core_path): |
138 raise error.TestFail('Core file was removed') | 157 raise error.TestFail('Core file was removed') |
139 | 158 |
140 | 159 |
| 160 def _test_sender_unknown_report_kind(self): |
| 161 self._set_sending(True) |
| 162 bad_report = self.create_fake_crash_dir_entry('fake.bad') |
| 163 result = self._call_sender_one_crash(report=bad_report) |
| 164 if (result['report_exists'] or |
| 165 result['rate_count'] != 0 or |
| 166 result['send_attempt'] or |
| 167 result['send_success'] or |
| 168 not 'Unknown report' in result['output']): |
| 169 raise error.TestFail('Error handling of unknown report kind failed') |
| 170 |
| 171 |
141 def _test_cron_runs(self): | 172 def _test_cron_runs(self): |
142 """Test sender runs successfully as part of the hourly cron job. | 173 """Test sender runs successfully as part of the hourly cron job. |
143 | 174 |
144 Assuming we've run test_sender_simple which shows that a minidump | 175 Assuming we've run test_sender_simple which shows that a minidump |
145 gets removed as part of sending, we run the cron job (which is | 176 gets removed as part of sending, we run the cron job (which is |
146 asynchronous) and wait for that file to be removed to just verify | 177 asynchronous) and wait for that file to be removed to just verify |
147 the job eventually runs the sender.""" | 178 the job eventually runs the sender.""" |
148 self._set_sending(True) | 179 self._set_sending(True) |
149 minidump = self._prepare_sender_one_crash(send_success=True, | 180 minidump = self._prepare_sender_one_crash(send_success=True, |
150 reports_enabled=True, | 181 reports_enabled=True, |
151 username='root', | 182 username='root', |
152 minidump=None) | 183 report=None) |
153 if not os.path.exists(minidump): | 184 if not os.path.exists(minidump): |
154 raise error.TestError('minidump not created') | 185 raise error.TestError('minidump not created') |
155 utils.system(_CRASH_SENDER_CRON_PATH) | 186 utils.system(_CRASH_SENDER_CRON_PATH) |
156 self._log_reader.set_start_by_current() | 187 self._log_reader.set_start_by_current() |
157 site_utils.poll_for_condition( | 188 site_utils.poll_for_condition( |
158 lambda: not os.path.exists(minidump), | 189 lambda: not os.path.exists(minidump), |
159 desc='minidump to be removed') | 190 desc='minidump to be removed') |
160 crash_sender_log = self._log_reader.get_logs() | 191 crash_sender_log = self._log_reader.get_logs() |
161 logging.debug('Contents of crash sender log: ' + crash_sender_log) | 192 logging.debug('Contents of crash sender log: ' + crash_sender_log) |
162 result = self._parse_sender_output(crash_sender_log) | 193 result = self._parse_sender_output(crash_sender_log) |
163 if not result['send_attempt'] or not result['send_success']: | 194 if not result['send_attempt'] or not result['send_success']: |
164 raise error.TestFail('Cron simple run test failed') | 195 raise error.TestFail('Cron simple run test failed') |
165 | 196 |
166 | 197 |
167 def run_once(self): | 198 def run_once(self): |
168 self.run_crash_tests([ | 199 self.run_crash_tests([ |
169 'sender_simple', | 200 'sender_simple_minidump', |
| 201 'sender_simple_kernel_crash', |
170 'sender_pausing', | 202 'sender_pausing', |
171 'sender_reports_disabled', | 203 'sender_reports_disabled', |
172 'sender_rate_limiting', | 204 'sender_rate_limiting', |
173 'sender_single_instance', | 205 'sender_single_instance', |
174 'sender_send_fails', | 206 'sender_send_fails', |
175 'sender_leaves_core_files', | 207 'sender_leaves_core_files', |
| 208 'sender_unknown_report_kind', |
176 'cron_runs']) | 209 'cron_runs']) |
OLD | NEW |