OLD | NEW |
1 # Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 # Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
2 # for details. All rights reserved. Use of this source code is governed by a | 2 # for details. All rights reserved. Use of this source code is governed by a |
3 # BSD-style license that can be found in the LICENSE file. | 3 # BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 # This file contains a set of utilities functions used by other Python-based | 5 # This file contains a set of utilities functions used by other Python-based |
6 # scripts. | 6 # scripts. |
7 | 7 |
| 8 from bots import bot_utils |
8 import commands | 9 import commands |
9 import datetime | 10 import datetime |
| 11 import glob |
10 import json | 12 import json |
11 import os | 13 import os |
12 import platform | 14 import platform |
13 import re | 15 import re |
14 import shutil | 16 import shutil |
15 import subprocess | 17 import subprocess |
| 18 import sys |
| 19 import tarfile |
16 import tempfile | 20 import tempfile |
17 import sys | 21 import uuid |
| 22 |
| 23 try: |
| 24 # Not available on Windows. |
| 25 import resource |
| 26 except: |
| 27 pass |
18 | 28 |
19 class Version(object): | 29 class Version(object): |
20 def __init__(self, channel, major, minor, patch, prerelease, | 30 def __init__(self, channel, major, minor, patch, prerelease, |
21 prerelease_patch): | 31 prerelease_patch): |
22 self.channel = channel | 32 self.channel = channel |
23 self.major = major | 33 self.major = major |
24 self.minor = minor | 34 self.minor = minor |
25 self.patch = patch | 35 self.patch = patch |
26 self.prerelease = prerelease | 36 self.prerelease = prerelease |
27 self.prerelease_patch = prerelease_patch | 37 self.prerelease_patch = prerelease_patch |
(...skipping 622 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
650 | 660 |
651 def __enter__(self): | 661 def __enter__(self): |
652 self._old_cwd = os.getcwd() | 662 self._old_cwd = os.getcwd() |
653 print "Enter directory = ", self._working_directory | 663 print "Enter directory = ", self._working_directory |
654 os.chdir(self._working_directory) | 664 os.chdir(self._working_directory) |
655 | 665 |
656 def __exit__(self, *_): | 666 def __exit__(self, *_): |
657 print "Enter directory = ", self._old_cwd | 667 print "Enter directory = ", self._old_cwd |
658 os.chdir(self._old_cwd) | 668 os.chdir(self._old_cwd) |
659 | 669 |
| 670 # This class finds and archives all core.* files from the current working |
| 671 # directory and all binaries copied by UnexpectedCrashDumpArchiver into |
| 672 # the current working directory (see tools/testing/dart/test_progress.dart). |
| 673 class CoreDumpArchiver(object): |
| 674 def __init__(self, args): |
| 675 self._enabled = '--copy-coredumps' in args and GuessOS() == 'linux' |
| 676 self._search_dir = os.getcwd() |
| 677 self._bucket = 'dart-temp-crash-archive' |
| 678 self._old_limits = None |
| 679 |
| 680 def __enter__(self): |
| 681 if not self._enabled: |
| 682 return |
| 683 |
| 684 # Cleanup any stale coredumps |
| 685 coredumps = self._find_coredumps() |
| 686 if coredumps: |
| 687 print "WARNING: Found stale coredumps, removing" |
| 688 MarkCurrentStepWarning() |
| 689 self._remove_coredumps(coredumps) |
| 690 |
| 691 self._old_limits = resource.getrlimit(resource.RLIMIT_CORE) |
| 692 |
| 693 # Bump core limits to unlimited if core_pattern is correctly configured. |
| 694 if self._check_core_dump_pattern(fatal=False): |
| 695 resource.setrlimit(resource.RLIMIT_CORE, (-1, -1)) |
| 696 |
| 697 def __exit__(self, *_): |
| 698 if not self._enabled: |
| 699 return |
| 700 |
| 701 # Restore old core limit. |
| 702 resource.setrlimit(resource.RLIMIT_CORE, self._old_limits) |
| 703 |
| 704 # Check that kernel was correctly configured to use core.%p |
| 705 # core_pattern. |
| 706 self._check_core_dump_pattern(fatal=True) |
| 707 |
| 708 coredumps = self._find_coredumps() |
| 709 if coredumps: |
| 710 # If we get a ton of crashes, only archive 10 dumps. |
| 711 archive_coredumps = coredumps[:10] |
| 712 print 'Archiving coredumps: %s' % ', '.join(archive_coredumps) |
| 713 sys.stdout.flush() |
| 714 self._archive(archive_coredumps) |
| 715 self._remove_coredumps(coredumps) |
| 716 coredumps = self._find_coredumps() |
| 717 assert not coredumps |
| 718 |
| 719 def _find_coredumps(self): |
| 720 return glob.glob(os.path.join(self._search_dir, 'core.*')) |
| 721 |
| 722 def _remove_coredumps(self, coredumps): |
| 723 for name in coredumps: |
| 724 os.unlink(name) |
| 725 |
| 726 def _archive(self, coredumps): |
| 727 gsutil = bot_utils.GSUtil() |
| 728 storage_path = '%s/%s/' % (self._bucket, uuid.uuid4()) |
| 729 gs_prefix = 'gs://%s' % storage_path |
| 730 http_prefix = 'https://storage.cloud.google.com/%s' % storage_path |
| 731 |
| 732 for core in coredumps: |
| 733 # Sanitize the name: actual cores follow 'core.%d' pattern, crashed |
| 734 # binaries are copied next to cores and named 'core.<binary_name>'. |
| 735 suffix = os.path.basename(core).split('.')[1] |
| 736 try: |
| 737 # Check if suffix is an integer - in this case it's an actual core. |
| 738 clean_name = 'core.%d' % int(suffix) |
| 739 except: |
| 740 # This is not a coredump but a crashed binary. |
| 741 clean_name = suffix |
| 742 |
| 743 tarname = '%s.tar.gz' % clean_name |
| 744 |
| 745 # Create a .tar.gz archive out of a crash folder that contains |
| 746 # both binary and the core dump. |
| 747 tar = tarfile.open(tarname, mode='w:gz') |
| 748 tar.add(core, arcname=clean_name) |
| 749 tar.close() |
| 750 |
| 751 # Remove / from absolute path to not have // in gs path. |
| 752 gs_url = '%s%s' % (gs_prefix, tarname) |
| 753 http_url = '%s%s' % (http_prefix, tarname) |
| 754 |
| 755 try: |
| 756 gsutil.upload(tarname, gs_url) |
| 757 print '@@@STEP_LOG_LINE@coredumps@%s (%s)@@@' % (gs_url, http_url) |
| 758 except Exception as error: |
| 759 message = "Failed to upload coredump %s, error: %s" % (tarname, error) |
| 760 print '@@@STEP_LOG_LINE@coredumps@%s@@@' % message |
| 761 |
| 762 os.unlink(tarname) |
| 763 |
| 764 print '@@@STEP_LOG_END@coredumps@@@' |
| 765 MarkCurrentStepWarning() |
| 766 |
| 767 def _check_core_dump_pattern(self, fatal=False): |
| 768 core_pattern_file = '/proc/sys/kernel/core_pattern' |
| 769 core_pattern = open(core_pattern_file).read() |
| 770 |
| 771 expected_core_pattern = 'core.%p' |
| 772 if core_pattern.strip() != expected_core_pattern: |
| 773 if fatal: |
| 774 message = ("Invalid core_pattern configuration. " |
| 775 "The configuration of core dump handling is *not* correct for " |
| 776 "a buildbot. The content of {0} must be '{1}' instead of '{2}'." |
| 777 .format(core_pattern_file, expected_core_pattern, core_pattern)) |
| 778 raise Exception(message) |
| 779 else: |
| 780 return False |
| 781 return True |
| 782 |
| 783 def MarkCurrentStepWarning(): |
| 784 print "@@@STEP_WARNINGS@@@" |
| 785 sys.stdout.flush() |
660 | 786 |
661 if __name__ == "__main__": | 787 if __name__ == "__main__": |
662 import sys | 788 import sys |
663 Main() | 789 Main() |
OLD | NEW |