OLD | NEW |
1 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved. | 1 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 import grp, logging, os, pwd, re, stat, subprocess | 5 import grp, logging, os, pwd, re, stat, subprocess |
6 from signal import SIGSEGV | 6 from signal import SIGSEGV |
7 from autotest_lib.client.bin import site_crash_test, site_utils, test | 7 from autotest_lib.client.bin import site_crash_test, site_utils, test |
8 from autotest_lib.client.common_lib import error, utils | 8 from autotest_lib.client.common_lib import error, utils |
9 | 9 |
10 _CORE_PATTERN = '/proc/sys/kernel/core_pattern' | 10 _CORE_PATTERN = '/proc/sys/kernel/core_pattern' |
(...skipping 34 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
45 if output != 'core': | 45 if output != 'core': |
46 raise error.TestFail('core pattern should have been core, not %s' % | 46 raise error.TestFail('core pattern should have been core, not %s' % |
47 output) | 47 output) |
48 | 48 |
49 | 49 |
50 def _prepare_crasher(self): | 50 def _prepare_crasher(self): |
51 """Extract the crasher and set its permissions. | 51 """Extract the crasher and set its permissions. |
52 | 52 |
53 crasher is only gzipped to subvert Portage stripping. | 53 crasher is only gzipped to subvert Portage stripping. |
54 """ | 54 """ |
55 self._crasher_path = { | 55 self._crasher_path = os.path.join(self.srcdir, 'crasher_nobreakpad') |
56 True: os.path.join(self.srcdir, 'crasher_breakpad'), | |
57 False: os.path.join(self.srcdir, 'crasher_nobreakpad') | |
58 } | |
59 utils.system('cd %s; tar xzf crasher.tgz' % | 56 utils.system('cd %s; tar xzf crasher.tgz' % |
60 self.srcdir) | 57 self.srcdir) |
61 | 58 |
62 | 59 |
63 def _populate_symbols(self): | 60 def _populate_symbols(self): |
64 """Set up Breakpad's symbol structure. | 61 """Set up Breakpad's symbol structure. |
65 | 62 |
66 Breakpad's minidump processor expects symbols to be in a directory | 63 Breakpad's minidump processor expects symbols to be in a directory |
67 hierarchy: | 64 hierarchy: |
68 <symbol-root>/<module_name>/<file_id>/<module_name>.sym | 65 <symbol-root>/<module_name>/<file_id>/<module_name>.sym |
69 """ | 66 """ |
70 # Dump the symbols from the crasher | 67 # Dump the symbols from the crasher |
71 self._symbol_dir = os.path.join(self.srcdir, 'symbols') | 68 self._symbol_dir = os.path.join(self.srcdir, 'symbols') |
72 utils.system('rm -rf %s' % self._symbol_dir) | 69 utils.system('rm -rf %s' % self._symbol_dir) |
73 os.mkdir(self._symbol_dir) | 70 os.mkdir(self._symbol_dir) |
74 | 71 |
75 for with_breakpad in [True, False]: | 72 basename = os.path.basename(self._crasher_path) |
76 basename = os.path.basename(self._crasher_path[with_breakpad]) | 73 utils.system('/usr/bin/dump_syms %s > %s.sym' % |
77 utils.system('/usr/bin/dump_syms %s > %s.sym' % | 74 (self._crasher_path, |
78 (self._crasher_path[with_breakpad], | 75 basename)) |
79 basename)) | 76 sym_name = '%s.sym' % basename |
80 sym_name = '%s.sym' % basename | 77 symbols = utils.read_file(sym_name) |
81 symbols = utils.read_file(sym_name) | 78 # First line should be like: |
82 # First line should be like: | 79 # MODULE Linux x86 7BC3323FBDBA2002601FA5BA3186D6540 crasher_XXX |
83 # MODULE Linux x86 7BC3323FBDBA2002601FA5BA3186D6540 crasher_XXX | 80 # or |
84 # or | 81 # MODULE Linux arm C2FE4895B203D87DD4D9227D5209F7890 crasher_XXX |
85 # MODULE Linux arm C2FE4895B203D87DD4D9227D5209F7890 crasher_XXX | 82 first_line = symbols.split('\n')[0] |
86 first_line = symbols.split('\n')[0] | 83 tokens = first_line.split() |
87 tokens = first_line.split() | 84 if tokens[0] != 'MODULE' or tokens[1] != 'Linux': |
88 if tokens[0] != 'MODULE' or tokens[1] != 'Linux': | 85 raise error.TestError('Unexpected symbols format: %s', |
89 raise error.TestError('Unexpected symbols format: %s', | 86 first_line) |
90 first_line) | 87 file_id = tokens[3] |
91 file_id = tokens[3] | 88 target_dir = os.path.join(self._symbol_dir, basename, file_id) |
92 target_dir = os.path.join(self._symbol_dir, basename, file_id) | 89 os.makedirs(target_dir) |
93 os.makedirs(target_dir) | 90 os.rename(sym_name, os.path.join(target_dir, sym_name)) |
94 os.rename(sym_name, os.path.join(target_dir, sym_name)) | |
95 | 91 |
96 | 92 |
97 def _verify_stack(self, stack, basename, from_crash_reporter): | 93 def _verify_stack(self, stack, basename, from_crash_reporter): |
98 logging.debug('Crash stackwalk was: %s' % stack) | 94 logging.debug('Crash stackwalk was: %s' % stack) |
99 | 95 |
100 # Should identify cause as SIGSEGV at address 0x16 | 96 # Should identify cause as SIGSEGV at address 0x16 |
101 match = re.search(r'Crash reason:\s+(.*)', stack) | 97 match = re.search(r'Crash reason:\s+(.*)', stack) |
102 expected_address = '0x16' | 98 expected_address = '0x16' |
103 if from_crash_reporter: | 99 if from_crash_reporter: |
104 # We cannot yet determine the crash address when coming | 100 # We cannot yet determine the crash address when coming |
(...skipping 13 matching lines...) Expand all Loading... |
118 # Should identify recursion line which is on the stack | 114 # Should identify recursion line which is on the stack |
119 # for 15 levels | 115 # for 15 levels |
120 if not ('15 %s!recbomb(int) [bomb.cc : 12 ' % basename) in stack: | 116 if not ('15 %s!recbomb(int) [bomb.cc : 12 ' % basename) in stack: |
121 raise error.TestFail('Did not show recursion line on stack') | 117 raise error.TestFail('Did not show recursion line on stack') |
122 | 118 |
123 # Should identify main line | 119 # Should identify main line |
124 if not ('16 %s!main [crasher.cc : 21 ' % basename) in stack: | 120 if not ('16 %s!main [crasher.cc : 21 ' % basename) in stack: |
125 raise error.TestFail('Did not show main on stack') | 121 raise error.TestFail('Did not show main on stack') |
126 | 122 |
127 | 123 |
128 def _run_crasher_process(self, username, with_breakpad, cause_crash=True): | 124 def _run_crasher_process(self, username, cause_crash=True): |
129 """Runs the crasher process. | 125 """Runs the crasher process. |
130 | 126 |
131 Args: | 127 Args: |
132 username: runs as given user | 128 username: runs as given user |
133 with_breakpad: run crasher that has breakpad (-lcrash) linked in | |
134 extra_args: additional parameters to pass to crasher process | 129 extra_args: additional parameters to pass to crasher process |
135 | 130 |
136 Returns: | 131 Returns: |
137 A dictionary with keys: | 132 A dictionary with keys: |
138 returncode: return code of the crasher | 133 returncode: return code of the crasher |
139 crashed: did the crasher return segv error code | 134 crashed: did the crasher return segv error code |
140 crash_reporter_caught: did crash_reporter catch a segv | 135 crash_reporter_caught: did crash_reporter catch a segv |
141 output: stderr/stdout output of the crasher process | 136 output: stderr/stdout output of the crasher process |
142 """ | 137 """ |
143 self._prepare_crasher() | 138 self._prepare_crasher() |
144 self._populate_symbols() | 139 self._populate_symbols() |
145 | 140 |
146 if username != 'root': | 141 if username != 'root': |
147 crasher_command = ['su', username, '-c'] | 142 crasher_command = ['su', username, '-c'] |
148 expected_result = 128 + SIGSEGV | 143 expected_result = 128 + SIGSEGV |
149 else: | 144 else: |
150 crasher_command = [] | 145 crasher_command = [] |
151 expected_result = -SIGSEGV | 146 expected_result = -SIGSEGV |
152 | 147 |
153 crasher_command.append(self._crasher_path[with_breakpad]) | 148 crasher_command.append(self._crasher_path) |
154 basename = os.path.basename(self._crasher_path[with_breakpad]) | 149 basename = os.path.basename(self._crasher_path) |
155 if not cause_crash: | 150 if not cause_crash: |
156 crasher_command.append('--nocrash') | 151 crasher_command.append('--nocrash') |
157 crasher = subprocess.Popen(crasher_command, | 152 crasher = subprocess.Popen(crasher_command, |
158 stdout=subprocess.PIPE, | 153 stdout=subprocess.PIPE, |
159 stderr=subprocess.PIPE) | 154 stderr=subprocess.PIPE) |
160 output = crasher.communicate()[1] | 155 output = crasher.communicate()[1] |
161 logging.debug('Output from %s: %s' % | 156 logging.debug('Output from %s: %s' % |
162 (self._crasher_path[with_breakpad], output)) | 157 (self._crasher_path, output)) |
163 | 158 |
164 # Grab the pid from the process output. We can't just use | 159 # Grab the pid from the process output. We can't just use |
165 # crasher.pid unfortunately because that may be the PID of su. | 160 # crasher.pid unfortunately because that may be the PID of su. |
166 match = re.search(r'pid=(\d+)', output) | 161 match = re.search(r'pid=(\d+)', output) |
167 if not match: | 162 if not match: |
168 raise error.TestFail('Could not find pid output from crasher: %s' % | 163 raise error.TestFail('Could not find pid output from crasher: %s' % |
169 output) | 164 output) |
170 pid = int(match.group(1)) | 165 pid = int(match.group(1)) |
171 | 166 |
172 expected_message = ('Received crash notification for ' | 167 expected_message = ('Received crash notification for ' |
(...skipping 65 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
238 raise error.TestFail('Minidump not sent properly') | 233 raise error.TestFail('Minidump not sent properly') |
239 if will_syslog_give_name: | 234 if will_syslog_give_name: |
240 if result['exec_name'] != crasher_basename: | 235 if result['exec_name'] != crasher_basename: |
241 raise error.TestFail('Executable name incorrect') | 236 raise error.TestFail('Executable name incorrect') |
242 if result['report_kind'] != 'minidump': | 237 if result['report_kind'] != 'minidump': |
243 raise error.TestFail('Expected a minidump report') | 238 raise error.TestFail('Expected a minidump report') |
244 if result['report_name'] != minidump_path: | 239 if result['report_name'] != minidump_path: |
245 raise error.TestFail('Sent the wrong minidump report') | 240 raise error.TestFail('Sent the wrong minidump report') |
246 | 241 |
247 | 242 |
248 def _check_crashing_process(self, username, with_breakpad): | 243 def _check_crashing_process(self, username): |
249 self._log_reader.set_start_by_current() | 244 self._log_reader.set_start_by_current() |
250 | 245 |
251 result = self._run_crasher_process(username, with_breakpad) | 246 result = self._run_crasher_process(username) |
252 | 247 |
253 if not result['crashed']: | 248 if not result['crashed']: |
254 raise error.TestFail('crasher did not do its job of crashing: %d' % | 249 raise error.TestFail('crasher did not do its job of crashing: %d' % |
255 result['returncode']) | 250 result['returncode']) |
256 | 251 |
257 if not result['crash_reporter_caught']: | 252 if not result['crash_reporter_caught']: |
258 logging.debug('Messages that should have included segv: %s' % | 253 logging.debug('Messages that should have included segv: %s' % |
259 self._log_reader.get_logs()) | 254 self._log_reader.get_logs()) |
260 raise error.TestFail('Did not find segv message') | 255 raise error.TestFail('Did not find segv message') |
261 | 256 |
262 crash_dir = self._get_crash_dir(username) | 257 crash_dir = self._get_crash_dir(username) |
263 crash_contents = os.listdir(crash_dir) | 258 crash_contents = os.listdir(crash_dir) |
264 basename = os.path.basename(self._crasher_path[with_breakpad]) | 259 basename = os.path.basename(self._crasher_path) |
265 | 260 |
266 breakpad_minidump = None | 261 breakpad_minidump = None |
267 crash_reporter_minidump = None | 262 crash_reporter_minidump = None |
268 | 263 |
269 self._check_crash_directory_permissions(crash_dir) | 264 self._check_crash_directory_permissions(crash_dir) |
270 | 265 |
271 logging.debug('Contents in %s: %s' % (crash_dir, crash_contents)) | 266 logging.debug('Contents in %s: %s' % (crash_dir, crash_contents)) |
272 | 267 |
273 for filename in crash_contents: | 268 for filename in crash_contents: |
274 if filename.endswith('.core'): | 269 if filename.endswith('.core'): |
275 # Ignore core files. We'll test them later. | 270 # Ignore core files. We'll test them later. |
276 pass | 271 pass |
277 elif filename.startswith(basename): | 272 elif filename.startswith(basename): |
278 # This appears to be a minidump created by the crash reporter. | 273 # This appears to be a minidump created by the crash reporter. |
279 if not crash_reporter_minidump is None: | 274 if not crash_reporter_minidump is None: |
280 raise error.TestFail('Crash reporter wrote multiple ' | 275 raise error.TestFail('Crash reporter wrote multiple ' |
281 'minidumps') | 276 'minidumps') |
282 crash_reporter_minidump = os.path.join(crash_dir, filename) | 277 crash_reporter_minidump = os.path.join(crash_dir, filename) |
283 else: | 278 else: |
284 # This appears to be a breakpad created minidump. | 279 # This appears to be a breakpad created minidump. |
285 if not breakpad_minidump is None: | 280 if not breakpad_minidump is None: |
286 raise error.TestFail('Breakpad wrote multimpe minidumps') | 281 raise error.TestFail('Breakpad wrote multimpe minidumps') |
287 breakpad_minidump = os.path.join(crash_dir, filename) | 282 breakpad_minidump = os.path.join(crash_dir, filename) |
288 | 283 |
289 if with_breakpad and not breakpad_minidump: | 284 if breakpad_minidump: |
290 raise error.TestFail('%s did not generate breakpad minidump' % | |
291 basename) | |
292 | |
293 if not with_breakpad and breakpad_minidump: | |
294 raise error.TestFail('%s did generate breakpad minidump' % basename) | 285 raise error.TestFail('%s did generate breakpad minidump' % basename) |
295 | 286 |
296 if not crash_reporter_minidump: | 287 if not crash_reporter_minidump: |
297 raise error.TestFail('crash reporter did not generate minidump') | 288 raise error.TestFail('crash reporter did not generate minidump') |
298 | 289 |
299 if not self._log_reader.can_find('Stored minidump to ' + | 290 if not self._log_reader.can_find('Stored minidump to ' + |
300 crash_reporter_minidump): | 291 crash_reporter_minidump): |
301 raise error.TestFail('crash reporter did not announce minidump') | 292 raise error.TestFail('crash reporter did not announce minidump') |
302 | 293 |
303 # By default test sending the crash_reporter minidump unless there | 294 # By default test sending the crash_reporter minidump unless there |
(...skipping 22 matching lines...) Expand all Loading... |
326 | 317 |
327 self._check_generated_minidump_sending(send_minidump, | 318 self._check_generated_minidump_sending(send_minidump, |
328 username, | 319 username, |
329 basename, | 320 basename, |
330 will_syslog_give_name) | 321 will_syslog_give_name) |
331 | 322 |
332 def _test_no_crash(self): | 323 def _test_no_crash(self): |
333 """Test a program linked against libcrash_dumper can exit normally.""" | 324 """Test a program linked against libcrash_dumper can exit normally.""" |
334 self._log_reader.set_start_by_current() | 325 self._log_reader.set_start_by_current() |
335 result = self._run_crasher_process(username='root', | 326 result = self._run_crasher_process(username='root', |
336 with_breakpad=True, | |
337 cause_crash=False) | 327 cause_crash=False) |
338 if (result['crashed'] or | 328 if (result['crashed'] or |
339 result['crash_reporter_caught'] or | 329 result['crash_reporter_caught'] or |
340 result['returncode'] != 0): | 330 result['returncode'] != 0): |
341 raise error.TestFail('Normal exit of program with dumper failed') | 331 raise error.TestFail('Normal exit of program with dumper failed') |
342 | 332 |
343 | 333 |
344 def _test_chronos_breakpad_crasher(self): | |
345 """Test a user space crash when running as chronos is handled.""" | |
346 self._check_crashing_process('chronos', True) | |
347 | |
348 | |
349 def _test_chronos_nobreakpad_crasher(self): | 334 def _test_chronos_nobreakpad_crasher(self): |
350 """Test a user space crash when running as chronos is handled.""" | 335 """Test a user space crash when running as chronos is handled.""" |
351 self._check_crashing_process('chronos', False) | 336 self._check_crashing_process('chronos') |
352 | |
353 | |
354 def _test_root_breakpad_crasher(self): | |
355 """Test a user space crash when running as root is handled.""" | |
356 self._check_crashing_process('root', True) | |
357 | 337 |
358 | 338 |
359 def _test_root_nobreakpad_crasher(self): | 339 def _test_root_nobreakpad_crasher(self): |
360 """Test a user space crash when running as root is handled.""" | 340 """Test a user space crash when running as root is handled.""" |
361 self._check_crashing_process('root', False) | 341 self._check_crashing_process('root') |
362 | 342 |
363 | 343 |
364 def _test_max_enqueued_crashes(self): | 344 def _test_max_enqueued_crashes(self): |
365 """Test that _MAX_CRASH_DIRECTORY_SIZE is enforced.""" | 345 """Test that _MAX_CRASH_DIRECTORY_SIZE is enforced.""" |
366 self._log_reader.set_start_by_current() | 346 self._log_reader.set_start_by_current() |
367 username = 'root' | 347 username = 'root' |
368 | 348 |
369 crash_dir = self._get_crash_dir(username) | 349 crash_dir = self._get_crash_dir(username) |
370 full_message = ('Crash directory %s already full with %d pending ' | 350 full_message = ('Crash directory %s already full with %d pending ' |
371 'reports' % (crash_dir, _MAX_CRASH_DIRECTORY_SIZE)) | 351 'reports' % (crash_dir, _MAX_CRASH_DIRECTORY_SIZE)) |
372 | 352 |
373 # Fill up the queue. | 353 # Fill up the queue. |
374 for i in range(0, _MAX_CRASH_DIRECTORY_SIZE): | 354 for i in range(0, _MAX_CRASH_DIRECTORY_SIZE): |
375 result = self._run_crasher_process(username, with_breakpad=False) | 355 result = self._run_crasher_process(username) |
376 if not result['crashed']: | 356 if not result['crashed']: |
377 raise error.TestFail('failure while setting up queue: %d' % | 357 raise error.TestFail('failure while setting up queue: %d' % |
378 result['returncode']) | 358 result['returncode']) |
379 if self._log_reader.can_find(full_message): | 359 if self._log_reader.can_find(full_message): |
380 raise error.TestFail('unexpected full message: ' + full_message) | 360 raise error.TestFail('unexpected full message: ' + full_message) |
381 | 361 |
382 crash_dir_size = len(os.listdir(crash_dir)) | 362 crash_dir_size = len(os.listdir(crash_dir)) |
383 # For debugging | 363 # For debugging |
384 utils.system('ls -l %s' % crash_dir) | 364 utils.system('ls -l %s' % crash_dir) |
385 logging.info('Crash directory had %d entries' % crash_dir_size) | 365 logging.info('Crash directory had %d entries' % crash_dir_size) |
386 | 366 |
387 # Crash a bunch more times, but make sure no new reports | 367 # Crash a bunch more times, but make sure no new reports |
388 # are enqueued. | 368 # are enqueued. |
389 for i in range(0, 10): | 369 for i in range(0, 10): |
390 self._log_reader.set_start_by_current() | 370 self._log_reader.set_start_by_current() |
391 result = self._run_crasher_process(username, with_breakpad=False) | 371 result = self._run_crasher_process(username) |
392 logging.info('New log messages: %s' % self._log_reader.get_logs()) | 372 logging.info('New log messages: %s' % self._log_reader.get_logs()) |
393 if not result['crashed']: | 373 if not result['crashed']: |
394 raise error.TestFail('failure after setting up queue: %d' % | 374 raise error.TestFail('failure after setting up queue: %d' % |
395 result['returncode']) | 375 result['returncode']) |
396 if not self._log_reader.can_find(full_message): | 376 if not self._log_reader.can_find(full_message): |
397 raise error.TestFail('expected full message: ' + full_message) | 377 raise error.TestFail('expected full message: ' + full_message) |
398 | 378 |
399 if crash_dir_size != len(os.listdir(crash_dir)): | 379 if crash_dir_size != len(os.listdir(crash_dir)): |
400 utils.system('ls -l %s' % crash_dir) | 380 utils.system('ls -l %s' % crash_dir) |
401 raise error.TestFail('expected no new files (now %d were %d)', | 381 raise error.TestFail('expected no new files (now %d were %d)', |
402 len(os.listdir(crash_dir)), | 382 len(os.listdir(crash_dir)), |
403 crash_dir_size) | 383 crash_dir_size) |
404 | 384 |
405 | 385 |
406 def _check_core_file_persisting(self, expect_persist): | 386 def _check_core_file_persisting(self, expect_persist): |
407 self._log_reader.set_start_by_current() | 387 self._log_reader.set_start_by_current() |
408 | 388 |
409 result = self._run_crasher_process('root', with_breakpad=False) | 389 result = self._run_crasher_process('root') |
410 | 390 |
411 if not result['crashed']: | 391 if not result['crashed']: |
412 raise error.TestFail('crasher did not crash') | 392 raise error.TestFail('crasher did not crash') |
413 | 393 |
414 crash_contents = os.listdir(self._get_crash_dir('root')) | 394 crash_contents = os.listdir(self._get_crash_dir('root')) |
415 | 395 |
416 logging.debug('Contents of crash directory: %s', crash_contents) | 396 logging.debug('Contents of crash directory: %s', crash_contents) |
417 logging.debug('Log messages: %s' % self._log_reader.get_logs()) | 397 logging.debug('Log messages: %s' % self._log_reader.get_logs()) |
418 | 398 |
419 if expect_persist: | 399 if expect_persist: |
(...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
463 os.system('umount /root') | 443 os.system('umount /root') |
464 | 444 |
465 | 445 |
466 # TODO(kmixter): Test crashing a process as ntp or some other | 446 # TODO(kmixter): Test crashing a process as ntp or some other |
467 # non-root, non-chronos user. | 447 # non-root, non-chronos user. |
468 | 448 |
469 def run_once(self): | 449 def run_once(self): |
470 self.run_crash_tests(['reporter_startup', | 450 self.run_crash_tests(['reporter_startup', |
471 'reporter_shutdown', | 451 'reporter_shutdown', |
472 'no_crash', | 452 'no_crash', |
473 'chronos_breakpad_crasher', | |
474 'chronos_nobreakpad_crasher', | 453 'chronos_nobreakpad_crasher', |
475 'root_breakpad_crasher', | |
476 'root_nobreakpad_crasher', | 454 'root_nobreakpad_crasher', |
477 'max_enqueued_crashes', | 455 'max_enqueued_crashes', |
478 'core_file_persists_in_debug', | 456 'core_file_persists_in_debug', |
479 'core_file_removed_in_production'], | 457 'core_file_removed_in_production'], |
480 initialize_crash_reporter = True) | 458 initialize_crash_reporter = True) |
OLD | NEW |