Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(446)

Side by Side Diff: client/site_tests/logging_CrashSender/logging_CrashSender.py

Issue 3114016: Added full E2E kcrash testing (Closed) Base URL: ssh://git@chromiumos-git//autotest.git
Patch Set: fixes Created 10 years, 4 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved. 1 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 import logging, os 5 import logging, os
6 from autotest_lib.client.bin import site_crash_test, site_utils, test 6 from autotest_lib.client.bin import site_crash_test, site_utils, test
7 from autotest_lib.client.common_lib import error, utils 7 from autotest_lib.client.common_lib import error, utils
8 8
9 _CRASH_SENDER_CRON_PATH = '/etc/cron.hourly/crash_sender.hourly' 9 _CRASH_SENDER_CRON_PATH = '/etc/cron.hourly/crash_sender.hourly'
10 _CRASH_SENDER_RUN_PATH = '/var/run/crash_sender.pid' 10 _CRASH_SENDER_RUN_PATH = '/var/run/crash_sender.pid'
11 _DAILY_RATE_LIMIT = 8 11 _DAILY_RATE_LIMIT = 8
12 _MIN_UNIQUE_TIMES = 4 12 _MIN_UNIQUE_TIMES = 4
13 _SECONDS_SEND_SPREAD = 3600 13 _SECONDS_SEND_SPREAD = 3600
14 14
15 15
16 class logging_CrashSender(site_crash_test.CrashTest): 16 class logging_CrashSender(site_crash_test.CrashTest):
17 version = 1 17 version = 1
18 18
19 19
20 def _test_sender_simple(self): 20 def _test_sender_simple_minidump(self):
21 """Test sending a single crash.""" 21 """Test sending a single minidump crash report."""
22 self._set_sending(True) 22 self._set_sending(True)
23 result = self._call_sender_one_crash() 23 result = self._call_sender_one_crash()
24 if (result['minidump_exists'] or 24 if (result['report_exists'] or
25 result['rate_count'] != 1 or 25 result['rate_count'] != 1 or
26 not result['send_attempt'] or 26 not result['send_attempt'] or
27 not result['send_success'] or 27 not result['send_success'] or
28 result['sleep_time'] < 0 or 28 result['sleep_time'] < 0 or
29 result['sleep_time'] >= _SECONDS_SEND_SPREAD): 29 result['sleep_time'] >= _SECONDS_SEND_SPREAD or
30 raise error.TestFail('Simple send failed') 30 result['report_kind'] != 'minidump' or
31 result['exec_name'] != 'fake'):
32 raise error.TestFail('Simple minidump send failed')
33
34
35 def _test_sender_simple_kernel_crash(self):
36 """Test sending a single kcrash report."""
37 self._set_sending(True)
38 kcrash_fake_report = self.create_fake_crash_dir_entry(
39 'kernel.today.kcrash')
40 result = self._call_sender_one_crash(report=kcrash_fake_report)
41 if (result['report_exists'] or
42 result['rate_count'] != 1 or
43 not result['send_attempt'] or
44 not result['send_success'] or
45 result['sleep_time'] < 0 or
46 result['sleep_time'] >= _SECONDS_SEND_SPREAD or
47 result['report_kind'] != 'kcrash' or
48 result['exec_name'] != 'kernel'):
49 raise error.TestFail('Simple kcrash send failed')
31 50
32 51
33 def _test_sender_pausing(self): 52 def _test_sender_pausing(self):
34 """Test the sender returns immediately when the pause file is present. 53 """Test the sender returns immediately when the pause file is present.
35 54
36 This is testing the sender's test functionality - if this regresses, 55 This is testing the sender's test functionality - if this regresses,
37 other tests can become flaky because the cron-started sender may run 56 other tests can become flaky because the cron-started sender may run
38 asynchronously to these tests.""" 57 asynchronously to these tests."""
39 self._set_sending(False) 58 self._set_sending(False)
40 result = self._call_sender_one_crash() 59 result = self._call_sender_one_crash()
41 if (not result['minidump_exists'] or 60 if (not result['report_exists'] or
42 not 'Exiting early due to' in result['output'] or 61 not 'Exiting early due to' in result['output'] or
43 result['send_attempt']): 62 result['send_attempt']):
44 raise error.TestFail('Sender did not pause') 63 raise error.TestFail('Sender did not pause')
45 64
46 65
47 def _test_sender_reports_disabled(self): 66 def _test_sender_reports_disabled(self):
48 """Test that when reporting is disabled, we don't send.""" 67 """Test that when reporting is disabled, we don't send."""
49 self._set_sending(True) 68 self._set_sending(True)
50 result = self._call_sender_one_crash(reports_enabled=False) 69 result = self._call_sender_one_crash(reports_enabled=False)
51 if (result['minidump_exists'] or 70 if (result['report_exists'] or
52 not 'Uploading is disabled' in result['output'] or 71 not 'Uploading is disabled' in result['output'] or
53 result['send_attempt']): 72 result['send_attempt']):
54 raise error.TestFail('Sender did not handle reports disabled') 73 raise error.TestFail('Sender did not handle reports disabled')
55 74
56 75
57 def _test_sender_rate_limiting(self): 76 def _test_sender_rate_limiting(self):
58 """Test the sender properly rate limits and sends with delay.""" 77 """Test the sender properly rate limits and sends with delay."""
59 self._set_sending(True) 78 self._set_sending(True)
60 sleep_times = [] 79 sleep_times = []
61 for i in range(1, _DAILY_RATE_LIMIT + 1): 80 for i in range(1, _DAILY_RATE_LIMIT + 1):
62 result = self._call_sender_one_crash() 81 result = self._call_sender_one_crash()
63 if not result['send_attempt'] or not result['send_success']: 82 if not result['send_attempt'] or not result['send_success']:
64 raise error.TestFail('Crash uploader did not send on #%d' % i) 83 raise error.TestFail('Crash uploader did not send on #%d' % i)
65 if result['rate_count'] != i: 84 if result['rate_count'] != i:
66 raise error.TestFail('Did not properly persist rate on #%d' % i) 85 raise error.TestFail('Did not properly persist rate on #%d' % i)
67 sleep_times.append(result['sleep_time']) 86 sleep_times.append(result['sleep_time'])
68 logging.debug('Sleeps between sending crashes were: %s' % sleep_times) 87 logging.debug('Sleeps between sending crashes were: %s' % sleep_times)
69 unique_times = {} 88 unique_times = {}
70 for i in range(0, _DAILY_RATE_LIMIT): 89 for i in range(0, _DAILY_RATE_LIMIT):
71 unique_times[sleep_times[i]] = True 90 unique_times[sleep_times[i]] = True
72 if len(unique_times) < _MIN_UNIQUE_TIMES: 91 if len(unique_times) < _MIN_UNIQUE_TIMES:
73 raise error.TestFail('Expected at least %d unique times: %s' % 92 raise error.TestFail('Expected at least %d unique times: %s' %
74 _MIN_UNIQUE_TIMES, sleep_times) 93 _MIN_UNIQUE_TIMES, sleep_times)
75 # Now the _DAILY_RATE_LIMIT ^ th send request should fail. 94 # Now the _DAILY_RATE_LIMIT ^ th send request should fail.
76 result = self._call_sender_one_crash() 95 result = self._call_sender_one_crash()
77 if (not result['minidump_exists'] or 96 if (not result['report_exists'] or
78 not 'Cannot send more crashes' in result['output'] or 97 not 'Cannot send more crashes' in result['output'] or
79 result['rate_count'] != _DAILY_RATE_LIMIT): 98 result['rate_count'] != _DAILY_RATE_LIMIT):
80 raise error.TestFail('Crash rate limiting did not take effect') 99 raise error.TestFail('Crash rate limiting did not take effect')
81 100
82 # Set one rate file a day earlier and verify can send 101 # Set one rate file a day earlier and verify can send
83 rate_files = os.listdir(self._CRASH_SENDER_RATE_DIR) 102 rate_files = os.listdir(self._CRASH_SENDER_RATE_DIR)
84 rate_path = os.path.join(self._CRASH_SENDER_RATE_DIR, rate_files[0]) 103 rate_path = os.path.join(self._CRASH_SENDER_RATE_DIR, rate_files[0])
85 statinfo = os.stat(rate_path) 104 statinfo = os.stat(rate_path)
86 os.utime(rate_path, (statinfo.st_atime, 105 os.utime(rate_path, (statinfo.st_atime,
87 statinfo.st_mtime - (60 * 60 * 25))) 106 statinfo.st_mtime - (60 * 60 * 25)))
88 utils.system('ls -l ' + self._CRASH_SENDER_RATE_DIR) 107 utils.system('ls -l ' + self._CRASH_SENDER_RATE_DIR)
89 result = self._call_sender_one_crash() 108 result = self._call_sender_one_crash()
90 if (not result['send_attempt'] or 109 if (not result['send_attempt'] or
91 not result['send_success'] or 110 not result['send_success'] or
92 result['rate_count'] != _DAILY_RATE_LIMIT): 111 result['rate_count'] != _DAILY_RATE_LIMIT):
93 raise error.TestFail('Crash not sent even after 25hrs pass') 112 raise error.TestFail('Crash not sent even after 25hrs pass')
94 113
95 114
96 def _test_sender_single_instance(self): 115 def _test_sender_single_instance(self):
97 """Test the sender fails to start when another instance is running. 116 """Test the sender fails to start when another instance is running.
98 117
99 Here we rely on the sender not checking the other running pid 118 Here we rely on the sender not checking the other running pid
100 is of the same instance. 119 is of the same instance.
101 """ 120 """
102 self._set_sending(True) 121 self._set_sending(True)
103 utils.open_write_close(_CRASH_SENDER_RUN_PATH, str(os.getpid())) 122 utils.open_write_close(_CRASH_SENDER_RUN_PATH, str(os.getpid()))
104 result = self._call_sender_one_crash() 123 result = self._call_sender_one_crash()
105 if (not 'Already running.' in result['output'] or 124 if (not 'Already running.' in result['output'] or
106 result['send_attempt'] or not result['minidump_exists']): 125 result['send_attempt'] or not result['report_exists']):
107 raise error.TestFail('Allowed multiple instances to run') 126 raise error.TestFail('Allowed multiple instances to run')
108 os.remove(_CRASH_SENDER_RUN_PATH) 127 os.remove(_CRASH_SENDER_RUN_PATH)
109 128
110 129
111 def _test_sender_send_fails(self): 130 def _test_sender_send_fails(self):
112 """Test that when the send fails we try again later.""" 131 """Test that when the send fails we try again later."""
113 self._set_sending(True) 132 self._set_sending(True)
114 result = self._call_sender_one_crash(send_success=False) 133 result = self._call_sender_one_crash(send_success=False)
115 if not result['send_attempt'] or result['send_success']: 134 if not result['send_attempt'] or result['send_success']:
116 raise error.TestError('Did not properly cause a send failure') 135 raise error.TestError('Did not properly cause a send failure')
117 if result['rate_count'] != 1: 136 if result['rate_count'] != 1:
118 raise error.TestFail('Did not count a failed send against rate ' 137 raise error.TestFail('Did not count a failed send against rate '
119 'limiting') 138 'limiting')
120 if not result['minidump_exists']: 139 if not result['report_exists']:
121 raise error.TestFail('Expected minidump to be saved for later ' 140 raise error.TestFail('Expected minidump to be saved for later '
122 'sending') 141 'sending')
123 142
124 143
125 def _test_sender_leaves_core_files(self): 144 def _test_sender_leaves_core_files(self):
126 """Test that a core file is left in the send directory. 145 """Test that a core file is left in the send directory.
127 146
128 Core files will only persist for developer/testing images. We 147 Core files will only persist for developer/testing images. We
129 should never remove such a file.""" 148 should never remove such a file."""
130 self._set_sending(True) 149 self._set_sending(True)
131 # Call prepare function to make sure the directory exists. 150 # Call prepare function to make sure the directory exists.
132 core_name = 'something.ending.with.core' 151 core_name = 'something.ending.with.core'
133 core_path = self._create_fake_crash_dir_entry(core_name) 152 core_path = self.create_fake_crash_dir_entry(core_name)
134 result = self._call_sender_one_crash() 153 result = self._call_sender_one_crash()
135 if not 'Ignoring core file.' in result['output']: 154 if not 'Ignoring core file.' in result['output']:
136 raise error.TestFail('Expected ignoring core file message') 155 raise error.TestFail('Expected ignoring core file message')
137 if not os.path.exists(core_path): 156 if not os.path.exists(core_path):
138 raise error.TestFail('Core file was removed') 157 raise error.TestFail('Core file was removed')
139 158
140 159
160 def _test_sender_unknown_report_kind(self):
161 self._set_sending(True)
162 bad_report = self.create_fake_crash_dir_entry('fake.bad')
163 result = self._call_sender_one_crash(report=bad_report)
164 if (result['report_exists'] or
165 result['rate_count'] != 0 or
166 result['send_attempt'] or
167 result['send_success'] or
168 not 'Unknown report' in result['output']):
169 raise error.TestFail('Error handling of unknown report kind failed')
170
171
141 def _test_cron_runs(self): 172 def _test_cron_runs(self):
142 """Test sender runs successfully as part of the hourly cron job. 173 """Test sender runs successfully as part of the hourly cron job.
143 174
144 Assuming we've run test_sender_simple which shows that a minidump 175 Assuming we've run test_sender_simple which shows that a minidump
145 gets removed as part of sending, we run the cron job (which is 176 gets removed as part of sending, we run the cron job (which is
146 asynchronous) and wait for that file to be removed to just verify 177 asynchronous) and wait for that file to be removed to just verify
147 the job eventually runs the sender.""" 178 the job eventually runs the sender."""
148 self._set_sending(True) 179 self._set_sending(True)
149 minidump = self._prepare_sender_one_crash(send_success=True, 180 minidump = self._prepare_sender_one_crash(send_success=True,
150 reports_enabled=True, 181 reports_enabled=True,
151 username='root', 182 username='root',
152 minidump=None) 183 report=None)
153 if not os.path.exists(minidump): 184 if not os.path.exists(minidump):
154 raise error.TestError('minidump not created') 185 raise error.TestError('minidump not created')
155 utils.system(_CRASH_SENDER_CRON_PATH) 186 utils.system(_CRASH_SENDER_CRON_PATH)
156 self._log_reader.set_start_by_current() 187 self._log_reader.set_start_by_current()
157 site_utils.poll_for_condition( 188 site_utils.poll_for_condition(
158 lambda: not os.path.exists(minidump), 189 lambda: not os.path.exists(minidump),
159 desc='minidump to be removed') 190 desc='minidump to be removed')
160 crash_sender_log = self._log_reader.get_logs() 191 crash_sender_log = self._log_reader.get_logs()
161 logging.debug('Contents of crash sender log: ' + crash_sender_log) 192 logging.debug('Contents of crash sender log: ' + crash_sender_log)
162 result = self._parse_sender_output(crash_sender_log) 193 result = self._parse_sender_output(crash_sender_log)
163 if not result['send_attempt'] or not result['send_success']: 194 if not result['send_attempt'] or not result['send_success']:
164 raise error.TestFail('Cron simple run test failed') 195 raise error.TestFail('Cron simple run test failed')
165 196
166 197
167 def run_once(self): 198 def run_once(self):
168 self.run_crash_tests([ 199 self.run_crash_tests([
169 'sender_simple', 200 'sender_simple_minidump',
201 'sender_simple_kernel_crash',
170 'sender_pausing', 202 'sender_pausing',
171 'sender_reports_disabled', 203 'sender_reports_disabled',
172 'sender_rate_limiting', 204 'sender_rate_limiting',
173 'sender_single_instance', 205 'sender_single_instance',
174 'sender_send_fails', 206 'sender_send_fails',
175 'sender_leaves_core_files', 207 'sender_leaves_core_files',
208 'sender_unknown_report_kind',
176 'cron_runs']) 209 'cron_runs'])
OLDNEW
« no previous file with comments | « client/bin/site_crash_test.py ('k') | client/site_tests/logging_KernelCrash/logging_KernelCrash.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698