OLD | NEW |
1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 """Functions specific to build slaves, shared by several buildbot scripts. | 5 """Functions specific to build slaves, shared by several buildbot scripts. |
6 """ | 6 """ |
7 | 7 |
8 import datetime | 8 import datetime |
9 import glob | 9 import glob |
10 import os | 10 import os |
11 import re | 11 import re |
12 import shutil | 12 import shutil |
13 import subprocess | |
14 import sys | 13 import sys |
15 import tempfile | 14 import tempfile |
16 import time | 15 import time |
17 | 16 |
18 from common import chromium_utils | 17 from common import chromium_utils |
19 from slave.bootstrap import ImportMasterConfigs # pylint: disable=W0611 | 18 from slave.bootstrap import ImportMasterConfigs # pylint: disable=W0611 |
20 from common.chromium_utils import GetActiveMaster # pylint: disable=W0611 | 19 from common.chromium_utils import GetActiveMaster # pylint: disable=W0611 |
21 | 20 |
22 # These codes used to distinguish true errors from script warnings. | 21 # These codes used to distinguish true errors from script warnings. |
23 ERROR_EXIT_CODE = 1 | 22 ERROR_EXIT_CODE = 1 |
24 WARNING_EXIT_CODE = 88 | 23 WARNING_EXIT_CODE = 88 |
25 | 24 |
26 | 25 |
27 # Regex matching git comment lines containing svn revision info. | |
28 GIT_SVN_ID_RE = re.compile(r'^git-svn-id: .*@([0-9]+) .*$') | |
29 # Regex for the master branch commit position. | |
30 GIT_CR_POS_RE = re.compile(r'^Cr-Commit-Position: refs/heads/master@{#(\d+)}$') | |
31 | |
32 | |
33 # Local errors. | 26 # Local errors. |
34 class PageHeapError(Exception): | 27 class PageHeapError(Exception): |
35 pass | 28 pass |
36 | 29 |
37 | 30 |
38 # Cache the path to gflags.exe. | 31 # Cache the path to gflags.exe. |
39 _gflags_exe = None | 32 _gflags_exe = None |
40 | 33 |
41 | 34 |
42 def SubversionExe(): | 35 def SubversionExe(): |
(...skipping 693 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
736 '--filesync', | 729 '--filesync', |
737 '--recurse-paths', | 730 '--recurse-paths', |
738 '--symlinks', | 731 '--symlinks', |
739 local_archive, | 732 local_archive, |
740 ] | 733 ] |
741 zip_cmd.extend(targets) | 734 zip_cmd.extend(targets) |
742 | 735 |
743 chromium_utils.RunCommand(zip_cmd) | 736 chromium_utils.RunCommand(zip_cmd) |
744 GSUtilCopy(local_archive, 'gs://%s/%s' % (bucket, archive)) | 737 GSUtilCopy(local_archive, 'gs://%s/%s' % (bucket, archive)) |
745 return 'https://storage.cloud.google.com/%s/%s' % (bucket, archive) | 738 return 'https://storage.cloud.google.com/%s/%s' % (bucket, archive) |
746 | |
747 | |
748 def GetPerfDashboardRevisions( | |
749 build_properties, main_revision, blink_revision, point_id=None): | |
750 """Fills in the same revisions fields that process_log_utils does.""" | |
751 return GetPerfDashboardRevisionsWithProperties( | |
752 build_properties.get('got_webrtc_revision'), | |
753 build_properties.get('got_v8_revision'), | |
754 build_properties.get('version'), | |
755 build_properties.get('git_revision'), | |
756 main_revision, blink_revision, point_id) | |
757 | |
758 | |
759 def GetPerfDashboardRevisionsWithProperties( | |
760 got_webrtc_revision, got_v8_revision, version, git_revision, main_revision, | |
761 blink_revision, point_id=None): | |
762 """Fills in the same revisions fields that process_log_utils does.""" | |
763 | |
764 versions = {} | |
765 versions['rev'] = main_revision | |
766 versions['webkit_rev'] = blink_revision | |
767 versions['webrtc_rev'] = got_webrtc_revision | |
768 versions['v8_rev'] = got_v8_revision | |
769 versions['ver'] = version | |
770 versions['git_revision'] = git_revision | |
771 versions['point_id'] = point_id | |
772 # There are a lot of "bad" revisions to check for, so clean them all up here. | |
773 for key in versions.keys(): | |
774 if not versions[key] or versions[key] == 'undefined': | |
775 del versions[key] | |
776 return versions | |
777 | |
778 | |
779 def GetMainRevision(build_properties, build_dir, revision=None): | |
780 """Return revision to use as the numerical x-value in the perf dashboard. | |
781 | |
782 This will be used as the value of "rev" in the data passed to | |
783 results_dashboard.SendResults. | |
784 | |
785 In order or priority, this function could return: | |
786 1. The value of the --revision flag (IF it can be parsed as an int). | |
787 2. The value of "got_revision_cp" in build properties. | |
788 3. An SVN number, git commit position, or git commit hash. | |
789 """ | |
790 if revision and revision.isdigit(): | |
791 return revision | |
792 commit_pos_num = _GetCommitPos(build_properties) | |
793 if commit_pos_num is not None: | |
794 return commit_pos_num | |
795 # TODO(sullivan,qyearsley): Don't fall back to _GetRevision if it returns | |
796 # a git commit, since this should be a numerical revision. Instead, abort | |
797 # and fail. | |
798 return GetRevision(os.path.dirname(os.path.abspath(build_dir))) | |
799 | |
800 | |
801 def GetBlinkRevision(build_dir, webkit_revision=None): | |
802 """ | |
803 TODO(eyaich): Blink's now folded into Chromium and doesn't have a separate | |
804 revision. Use main_revision and delete GetBlinkRevision and uses. | |
805 """ | |
806 if webkit_revision: | |
807 webkit_revision = webkit_revision | |
808 else: | |
809 try: | |
810 webkit_dir = chromium_utils.FindUpward( | |
811 os.path.abspath(build_dir), 'third_party', 'WebKit', 'Source') | |
812 webkit_revision = GetRevision(webkit_dir) | |
813 except Exception: | |
814 webkit_revision = None | |
815 return webkit_revision | |
816 | |
817 | |
818 def GetRevision(in_directory): | |
819 """Returns the SVN revision, git commit position, or git hash. | |
820 | |
821 Args: | |
822 in_directory: A directory in the repository to be checked. | |
823 | |
824 Returns: | |
825 An SVN revision as a string if the given directory is in a SVN repository, | |
826 or a git commit position number, or if that's not available, a git hash. | |
827 If all of that fails, an empty string is returned. | |
828 """ | |
829 import xml.dom.minidom | |
830 if not os.path.exists(os.path.join(in_directory, '.svn')): | |
831 if _IsGitDirectory(in_directory): | |
832 svn_rev = _GetGitCommitPosition(in_directory) | |
833 if svn_rev: | |
834 return svn_rev | |
835 return _GetGitRevision(in_directory) | |
836 else: | |
837 return '' | |
838 | |
839 # Note: Not thread safe: http://bugs.python.org/issue2320 | |
840 output = subprocess.Popen(['svn', 'info', '--xml'], | |
841 cwd=in_directory, | |
842 shell=(sys.platform == 'win32'), | |
843 stdout=subprocess.PIPE).communicate()[0] | |
844 try: | |
845 dom = xml.dom.minidom.parseString(output) | |
846 return dom.getElementsByTagName('entry')[0].getAttribute('revision') | |
847 except xml.parsers.expat.ExpatError: | |
848 return '' | |
849 return '' | |
850 | |
851 | |
852 def _GetCommitPos(build_properties): | |
853 """Extracts the commit position from the build properties, if its there.""" | |
854 if 'got_revision_cp' not in build_properties: | |
855 return None | |
856 commit_pos = build_properties['got_revision_cp'] | |
857 return int(re.search(r'{#(\d+)}', commit_pos).group(1)) | |
858 | |
859 | |
860 def _GetGitCommitPositionFromLog(log): | |
861 """Returns either the commit position or svn rev from a git log.""" | |
862 # Parse from the bottom up, in case the commit message embeds the message | |
863 # from a different commit (e.g., for a revert). | |
864 for r in [GIT_CR_POS_RE, GIT_SVN_ID_RE]: | |
865 for line in reversed(log.splitlines()): | |
866 m = r.match(line.strip()) | |
867 if m: | |
868 return m.group(1) | |
869 return None | |
870 | |
871 | |
872 def _GetGitCommitPosition(dir_path): | |
873 """Extracts the commit position or svn revision number of the HEAD commit.""" | |
874 git_exe = 'git.bat' if sys.platform.startswith('win') else 'git' | |
875 p = subprocess.Popen( | |
876 [git_exe, 'log', '-n', '1', '--pretty=format:%B', 'HEAD'], | |
877 cwd=dir_path, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) | |
878 (log, _) = p.communicate() | |
879 if p.returncode != 0: | |
880 return None | |
881 return _GetGitCommitPositionFromLog(log) | |
882 | |
883 | |
884 def _GetGitRevision(in_directory): | |
885 """Returns the git hash tag for the given directory. | |
886 | |
887 Args: | |
888 in_directory: The directory where git is to be run. | |
889 | |
890 Returns: | |
891 The git SHA1 hash string. | |
892 """ | |
893 git_exe = 'git.bat' if sys.platform.startswith('win') else 'git' | |
894 p = subprocess.Popen( | |
895 [git_exe, 'rev-parse', 'HEAD'], | |
896 cwd=in_directory, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) | |
897 (stdout, _) = p.communicate() | |
898 return stdout.strip() | |
899 | |
900 | |
901 def _IsGitDirectory(dir_path): | |
902 """Checks whether the given directory is in a git repository. | |
903 | |
904 Args: | |
905 dir_path: The directory path to be tested. | |
906 | |
907 Returns: | |
908 True if given directory is in a git repository, False otherwise. | |
909 """ | |
910 git_exe = 'git.bat' if sys.platform.startswith('win') else 'git' | |
911 with open(os.devnull, 'w') as devnull: | |
912 p = subprocess.Popen([git_exe, 'rev-parse', '--git-dir'], | |
913 cwd=dir_path, stdout=devnull, stderr=devnull) | |
914 return p.wait() == 0 | |
OLD | NEW |