| OLD | NEW |
| 1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 """Functions specific to build slaves, shared by several buildbot scripts. | 5 """Functions specific to build slaves, shared by several buildbot scripts. |
| 6 """ | 6 """ |
| 7 | 7 |
| 8 import datetime | 8 import datetime |
| 9 import glob | 9 import glob |
| 10 import os | 10 import os |
| 11 import re | 11 import re |
| 12 import shutil | 12 import shutil |
| 13 import subprocess | |
| 14 import sys | 13 import sys |
| 15 import tempfile | 14 import tempfile |
| 16 import time | 15 import time |
| 17 | 16 |
| 18 from common import chromium_utils | 17 from common import chromium_utils |
| 19 from slave.bootstrap import ImportMasterConfigs # pylint: disable=W0611 | 18 from slave.bootstrap import ImportMasterConfigs # pylint: disable=W0611 |
| 20 from common.chromium_utils import GetActiveMaster # pylint: disable=W0611 | 19 from common.chromium_utils import GetActiveMaster # pylint: disable=W0611 |
| 21 | 20 |
| 22 # These codes used to distinguish true errors from script warnings. | 21 # These codes used to distinguish true errors from script warnings. |
| 23 ERROR_EXIT_CODE = 1 | 22 ERROR_EXIT_CODE = 1 |
| 24 WARNING_EXIT_CODE = 88 | 23 WARNING_EXIT_CODE = 88 |
| 25 | 24 |
| 26 | 25 |
| 27 # Regex matching git comment lines containing svn revision info. | |
| 28 GIT_SVN_ID_RE = re.compile(r'^git-svn-id: .*@([0-9]+) .*$') | |
| 29 # Regex for the master branch commit position. | |
| 30 GIT_CR_POS_RE = re.compile(r'^Cr-Commit-Position: refs/heads/master@{#(\d+)}$') | |
| 31 | |
| 32 | |
| 33 # Local errors. | 26 # Local errors. |
| 34 class PageHeapError(Exception): | 27 class PageHeapError(Exception): |
| 35 pass | 28 pass |
| 36 | 29 |
| 37 | 30 |
| 38 # Cache the path to gflags.exe. | 31 # Cache the path to gflags.exe. |
| 39 _gflags_exe = None | 32 _gflags_exe = None |
| 40 | 33 |
| 41 | 34 |
| 42 def SubversionExe(): | 35 def SubversionExe(): |
| (...skipping 693 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 736 '--filesync', | 729 '--filesync', |
| 737 '--recurse-paths', | 730 '--recurse-paths', |
| 738 '--symlinks', | 731 '--symlinks', |
| 739 local_archive, | 732 local_archive, |
| 740 ] | 733 ] |
| 741 zip_cmd.extend(targets) | 734 zip_cmd.extend(targets) |
| 742 | 735 |
| 743 chromium_utils.RunCommand(zip_cmd) | 736 chromium_utils.RunCommand(zip_cmd) |
| 744 GSUtilCopy(local_archive, 'gs://%s/%s' % (bucket, archive)) | 737 GSUtilCopy(local_archive, 'gs://%s/%s' % (bucket, archive)) |
| 745 return 'https://storage.cloud.google.com/%s/%s' % (bucket, archive) | 738 return 'https://storage.cloud.google.com/%s/%s' % (bucket, archive) |
| 746 | |
| 747 | |
| 748 def GetPerfDashboardRevisions( | |
| 749 build_properties, main_revision, blink_revision, point_id=None): | |
| 750 """Fills in the same revisions fields that process_log_utils does.""" | |
| 751 return GetPerfDashboardRevisionsWithProperties( | |
| 752 build_properties.get('got_webrtc_revision'), | |
| 753 build_properties.get('got_v8_revision'), | |
| 754 build_properties.get('version'), | |
| 755 build_properties.get('git_revision'), | |
| 756 main_revision, blink_revision, point_id) | |
| 757 | |
| 758 | |
| 759 def GetPerfDashboardRevisionsWithProperties( | |
| 760 got_webrtc_revision, got_v8_revision, version, git_revision, main_revision, | |
| 761 blink_revision, point_id=None): | |
| 762 """Fills in the same revisions fields that process_log_utils does.""" | |
| 763 | |
| 764 versions = {} | |
| 765 versions['rev'] = main_revision | |
| 766 versions['webkit_rev'] = blink_revision | |
| 767 versions['webrtc_rev'] = got_webrtc_revision | |
| 768 versions['v8_rev'] = got_v8_revision | |
| 769 versions['ver'] = version | |
| 770 versions['git_revision'] = git_revision | |
| 771 versions['point_id'] = point_id | |
| 772 # There are a lot of "bad" revisions to check for, so clean them all up here. | |
| 773 for key in versions.keys(): | |
| 774 if not versions[key] or versions[key] == 'undefined': | |
| 775 del versions[key] | |
| 776 return versions | |
| 777 | |
| 778 | |
| 779 def GetMainRevision(build_properties, build_dir, revision=None): | |
| 780 """Return revision to use as the numerical x-value in the perf dashboard. | |
| 781 | |
| 782 This will be used as the value of "rev" in the data passed to | |
| 783 results_dashboard.SendResults. | |
| 784 | |
| 785 In order or priority, this function could return: | |
| 786 1. The value of the --revision flag (IF it can be parsed as an int). | |
| 787 2. The value of "got_revision_cp" in build properties. | |
| 788 3. An SVN number, git commit position, or git commit hash. | |
| 789 """ | |
| 790 if revision and revision.isdigit(): | |
| 791 return revision | |
| 792 commit_pos_num = _GetCommitPos(build_properties) | |
| 793 if commit_pos_num is not None: | |
| 794 return commit_pos_num | |
| 795 # TODO(sullivan,qyearsley): Don't fall back to _GetRevision if it returns | |
| 796 # a git commit, since this should be a numerical revision. Instead, abort | |
| 797 # and fail. | |
| 798 return GetRevision(os.path.dirname(os.path.abspath(build_dir))) | |
| 799 | |
| 800 | |
| 801 def GetBlinkRevision(build_dir, webkit_revision=None): | |
| 802 """ | |
| 803 TODO(eyaich): Blink's now folded into Chromium and doesn't have a separate | |
| 804 revision. Use main_revision and delete GetBlinkRevision and uses. | |
| 805 """ | |
| 806 if webkit_revision: | |
| 807 webkit_revision = webkit_revision | |
| 808 else: | |
| 809 try: | |
| 810 webkit_dir = chromium_utils.FindUpward( | |
| 811 os.path.abspath(build_dir), 'third_party', 'WebKit', 'Source') | |
| 812 webkit_revision = GetRevision(webkit_dir) | |
| 813 except Exception: | |
| 814 webkit_revision = None | |
| 815 return webkit_revision | |
| 816 | |
| 817 | |
| 818 def GetRevision(in_directory): | |
| 819 """Returns the SVN revision, git commit position, or git hash. | |
| 820 | |
| 821 Args: | |
| 822 in_directory: A directory in the repository to be checked. | |
| 823 | |
| 824 Returns: | |
| 825 An SVN revision as a string if the given directory is in a SVN repository, | |
| 826 or a git commit position number, or if that's not available, a git hash. | |
| 827 If all of that fails, an empty string is returned. | |
| 828 """ | |
| 829 import xml.dom.minidom | |
| 830 if not os.path.exists(os.path.join(in_directory, '.svn')): | |
| 831 if _IsGitDirectory(in_directory): | |
| 832 svn_rev = _GetGitCommitPosition(in_directory) | |
| 833 if svn_rev: | |
| 834 return svn_rev | |
| 835 return _GetGitRevision(in_directory) | |
| 836 else: | |
| 837 return '' | |
| 838 | |
| 839 # Note: Not thread safe: http://bugs.python.org/issue2320 | |
| 840 output = subprocess.Popen(['svn', 'info', '--xml'], | |
| 841 cwd=in_directory, | |
| 842 shell=(sys.platform == 'win32'), | |
| 843 stdout=subprocess.PIPE).communicate()[0] | |
| 844 try: | |
| 845 dom = xml.dom.minidom.parseString(output) | |
| 846 return dom.getElementsByTagName('entry')[0].getAttribute('revision') | |
| 847 except xml.parsers.expat.ExpatError: | |
| 848 return '' | |
| 849 return '' | |
| 850 | |
| 851 | |
| 852 def _GetCommitPos(build_properties): | |
| 853 """Extracts the commit position from the build properties, if its there.""" | |
| 854 if 'got_revision_cp' not in build_properties: | |
| 855 return None | |
| 856 commit_pos = build_properties['got_revision_cp'] | |
| 857 return int(re.search(r'{#(\d+)}', commit_pos).group(1)) | |
| 858 | |
| 859 | |
| 860 def _GetGitCommitPositionFromLog(log): | |
| 861 """Returns either the commit position or svn rev from a git log.""" | |
| 862 # Parse from the bottom up, in case the commit message embeds the message | |
| 863 # from a different commit (e.g., for a revert). | |
| 864 for r in [GIT_CR_POS_RE, GIT_SVN_ID_RE]: | |
| 865 for line in reversed(log.splitlines()): | |
| 866 m = r.match(line.strip()) | |
| 867 if m: | |
| 868 return m.group(1) | |
| 869 return None | |
| 870 | |
| 871 | |
| 872 def _GetGitCommitPosition(dir_path): | |
| 873 """Extracts the commit position or svn revision number of the HEAD commit.""" | |
| 874 git_exe = 'git.bat' if sys.platform.startswith('win') else 'git' | |
| 875 p = subprocess.Popen( | |
| 876 [git_exe, 'log', '-n', '1', '--pretty=format:%B', 'HEAD'], | |
| 877 cwd=dir_path, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) | |
| 878 (log, _) = p.communicate() | |
| 879 if p.returncode != 0: | |
| 880 return None | |
| 881 return _GetGitCommitPositionFromLog(log) | |
| 882 | |
| 883 | |
| 884 def _GetGitRevision(in_directory): | |
| 885 """Returns the git hash tag for the given directory. | |
| 886 | |
| 887 Args: | |
| 888 in_directory: The directory where git is to be run. | |
| 889 | |
| 890 Returns: | |
| 891 The git SHA1 hash string. | |
| 892 """ | |
| 893 git_exe = 'git.bat' if sys.platform.startswith('win') else 'git' | |
| 894 p = subprocess.Popen( | |
| 895 [git_exe, 'rev-parse', 'HEAD'], | |
| 896 cwd=in_directory, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) | |
| 897 (stdout, _) = p.communicate() | |
| 898 return stdout.strip() | |
| 899 | |
| 900 | |
| 901 def _IsGitDirectory(dir_path): | |
| 902 """Checks whether the given directory is in a git repository. | |
| 903 | |
| 904 Args: | |
| 905 dir_path: The directory path to be tested. | |
| 906 | |
| 907 Returns: | |
| 908 True if given directory is in a git repository, False otherwise. | |
| 909 """ | |
| 910 git_exe = 'git.bat' if sys.platform.startswith('win') else 'git' | |
| 911 with open(os.devnull, 'w') as devnull: | |
| 912 p = subprocess.Popen([git_exe, 'rev-parse', '--git-dir'], | |
| 913 cwd=dir_path, stdout=devnull, stderr=devnull) | |
| 914 return p.wait() == 0 | |
| OLD | NEW |