OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright 2014 The Chromium Authors. All rights reserved. | 2 # Copyright 2014 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 # TODO(hinoka): Use logging. | 6 # TODO(hinoka): Use logging. |
7 | 7 |
8 import cStringIO | 8 import cStringIO |
9 import codecs | 9 import codecs |
10 import collections | 10 import collections |
(...skipping 14 matching lines...) Expand all Loading... |
25 import urllib2 | 25 import urllib2 |
26 import urlparse | 26 import urlparse |
27 import uuid | 27 import uuid |
28 | 28 |
29 import os.path as path | 29 import os.path as path |
30 | 30 |
31 # How many bytes at a time to read from pipes. | 31 # How many bytes at a time to read from pipes. |
32 BUF_SIZE = 256 | 32 BUF_SIZE = 256 |
33 | 33 |
34 | 34 |
35 # TODO(luqui): This is a horrible hack to identify build_internal when build | |
36 # is a recipe dependency. bot_update should not be depending on internal, | |
37 # rather the arrow should go the other way (or just be destroyed). | |
38 def check_dir(name, dirs, default=None): | |
39 for d in dirs: | |
40 d = path.abspath(d) | |
41 if path.basename(d) == name and path.isdir(d): | |
42 return d | |
43 return default | |
44 | |
45 | |
46 # Define a bunch of directory paths. | 35 # Define a bunch of directory paths. |
47 # Relative to the current working directory. | 36 # Relative to the current working directory. |
48 CURRENT_DIR = path.abspath(os.getcwd()) | 37 CURRENT_DIR = path.abspath(os.getcwd()) |
49 BUILDER_DIR = path.dirname(CURRENT_DIR) | 38 BUILDER_DIR = path.dirname(CURRENT_DIR) |
50 SLAVE_DIR = path.dirname(BUILDER_DIR) | |
51 | 39 |
52 # Relative to this script's filesystem path. | 40 # Relative to this script's filesystem path. |
53 THIS_DIR = path.dirname(path.abspath(__file__)) | 41 THIS_DIR = path.dirname(path.abspath(__file__)) |
54 SCRIPTS_DIR = check_dir( | |
55 'scripts', [ | |
56 path.dirname(THIS_DIR), | |
57 path.join(SLAVE_DIR, '..', 'scripts'), | |
58 path.join(THIS_DIR, # resources | |
59 '..', # bot_update | |
60 '..', # recipe_modules | |
61 '..', # depot_tools | |
62 '..', # .recipe_deps | |
63 '..', # slave | |
64 '..', # scripts | |
65 '..', # build_internal | |
66 '..', # ROOT_DIR | |
67 'build', | |
68 'scripts'), | |
69 path.join(SLAVE_DIR, '..', 'build', 'scripts'), | |
70 ], default=path.dirname(THIS_DIR)) | |
71 BUILD_DIR = path.dirname(SCRIPTS_DIR) | |
72 ROOT_DIR = path.dirname(BUILD_DIR) | |
73 | 42 |
74 DEPOT_TOOLS_DIR = path.abspath(path.join(THIS_DIR, '..', '..', '..')) | 43 DEPOT_TOOLS_DIR = path.abspath(path.join(THIS_DIR, '..', '..', '..')) |
75 | 44 |
76 BUILD_INTERNAL_DIR = check_dir( | |
77 'build_internal', [ | |
78 path.join(ROOT_DIR, 'build_internal'), | |
79 path.join(ROOT_DIR, # .recipe_deps | |
80 path.pardir, # slave | |
81 path.pardir, # scripts | |
82 path.pardir), # build_internal | |
83 ]) | |
84 | |
85 | |
86 CHROMIUM_GIT_HOST = 'https://chromium.googlesource.com' | 45 CHROMIUM_GIT_HOST = 'https://chromium.googlesource.com' |
87 CHROMIUM_SRC_URL = CHROMIUM_GIT_HOST + '/chromium/src.git' | 46 CHROMIUM_SRC_URL = CHROMIUM_GIT_HOST + '/chromium/src.git' |
88 | 47 |
89 BRANCH_HEADS_REFSPEC = '+refs/branch-heads/*' | 48 BRANCH_HEADS_REFSPEC = '+refs/branch-heads/*' |
90 | 49 |
91 # Regular expression that matches a single commit footer line. | 50 # Regular expression that matches a single commit footer line. |
92 COMMIT_FOOTER_ENTRY_RE = re.compile(r'([^:]+):\s+(.+)') | 51 COMMIT_FOOTER_ENTRY_RE = re.compile(r'([^:]+):\s+(.+)') |
93 | 52 |
94 # Footer metadata keys for regular and gsubtreed mirrored commit positions. | 53 # Footer metadata keys for regular and gsubtreed mirrored commit positions. |
95 COMMIT_POSITION_FOOTER_KEY = 'Cr-Commit-Position' | 54 COMMIT_POSITION_FOOTER_KEY = 'Cr-Commit-Position' |
(...skipping 18 matching lines...) Expand all Loading... |
114 | 73 |
115 | 74 |
116 GCLIENT_TEMPLATE = """solutions = %(solutions)s | 75 GCLIENT_TEMPLATE = """solutions = %(solutions)s |
117 | 76 |
118 cache_dir = r%(cache_dir)s | 77 cache_dir = r%(cache_dir)s |
119 %(target_os)s | 78 %(target_os)s |
120 %(target_os_only)s | 79 %(target_os_only)s |
121 """ | 80 """ |
122 | 81 |
123 | 82 |
124 internal_data = {} | |
125 if BUILD_INTERNAL_DIR: | |
126 local_vars = {} | |
127 try: | |
128 execfile(os.path.join( | |
129 BUILD_INTERNAL_DIR, 'scripts', 'slave', 'bot_update_cfg.py'), | |
130 local_vars) | |
131 except Exception: | |
132 # Same as if BUILD_INTERNAL_DIR didn't exist in the first place. | |
133 print 'Warning: unable to read internal configuration file.' | |
134 print 'If this is an internal bot, this step may be erroneously inactive.' | |
135 internal_data = local_vars | |
136 | |
137 | |
138 | |
139 # How many times to try before giving up. | 83 # How many times to try before giving up. |
140 ATTEMPTS = 5 | 84 ATTEMPTS = 5 |
141 | 85 |
142 GIT_CACHE_PATH = path.join(DEPOT_TOOLS_DIR, 'git_cache.py') | 86 GIT_CACHE_PATH = path.join(DEPOT_TOOLS_DIR, 'git_cache.py') |
143 | 87 |
144 # Find the patch tool. | |
145 if sys.platform.startswith('win'): | |
146 if not BUILD_INTERNAL_DIR: | |
147 print 'Warning: could not find patch tool because there is no ' | |
148 print 'build_internal present.' | |
149 PATCH_TOOL = None | |
150 else: | |
151 PATCH_TOOL = path.join(BUILD_INTERNAL_DIR, 'tools', 'patch.EXE') | |
152 else: | |
153 PATCH_TOOL = '/usr/bin/patch' | |
154 | |
155 # If there is less than 100GB of disk space on the system, then we do | 88 # If there is less than 100GB of disk space on the system, then we do |
156 # a shallow checkout. | 89 # a shallow checkout. |
157 SHALLOW_CLONE_THRESHOLD = 100 * 1024 * 1024 * 1024 | 90 SHALLOW_CLONE_THRESHOLD = 100 * 1024 * 1024 * 1024 |
158 | 91 |
159 | 92 |
160 class SubprocessFailed(Exception): | 93 class SubprocessFailed(Exception): |
161 def __init__(self, message, code, output): | 94 def __init__(self, message, code, output): |
162 Exception.__init__(self, message) | 95 Exception.__init__(self, message) |
163 self.code = code | 96 self.code = code |
164 self.output = output | 97 self.output = output |
(...skipping 242 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
407 except SubprocessFailed as e: | 340 except SubprocessFailed as e: |
408 # Throw a GclientSyncFailed exception so we can catch this independently. | 341 # Throw a GclientSyncFailed exception so we can catch this independently. |
409 raise GclientSyncFailed(e.message, e.code, e.output) | 342 raise GclientSyncFailed(e.message, e.code, e.output) |
410 else: | 343 else: |
411 with open(gclient_output_file) as f: | 344 with open(gclient_output_file) as f: |
412 return json.load(f) | 345 return json.load(f) |
413 finally: | 346 finally: |
414 os.remove(gclient_output_file) | 347 os.remove(gclient_output_file) |
415 | 348 |
416 | 349 |
417 def gclient_runhooks(gyp_envs): | |
418 gclient_bin = 'gclient.bat' if sys.platform.startswith('win') else 'gclient' | |
419 env = dict([env_var.split('=', 1) for env_var in gyp_envs]) | |
420 call(gclient_bin, 'runhooks', env=env) | |
421 | |
422 | |
423 def gclient_revinfo(): | 350 def gclient_revinfo(): |
424 gclient_bin = 'gclient.bat' if sys.platform.startswith('win') else 'gclient' | 351 gclient_bin = 'gclient.bat' if sys.platform.startswith('win') else 'gclient' |
425 return call(gclient_bin, 'revinfo', '-a') or '' | 352 return call(gclient_bin, 'revinfo', '-a') or '' |
426 | 353 |
427 | 354 |
428 def create_manifest(): | 355 def create_manifest(): |
429 manifest = {} | 356 manifest = {} |
430 output = gclient_revinfo() | 357 output = gclient_revinfo() |
431 for line in output.strip().splitlines(): | 358 for line in output.strip().splitlines(): |
432 match = REVINFO_RE.match(line.strip()) | 359 match = REVINFO_RE.match(line.strip()) |
(...skipping 341 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
774 continue | 701 continue |
775 revision = get_target_revision(deps_name, deps_data.get('url', None), | 702 revision = get_target_revision(deps_name, deps_data.get('url', None), |
776 revisions) | 703 revisions) |
777 if not revision: | 704 if not revision: |
778 continue | 705 continue |
779 git('fetch', 'origin', cwd=deps_name) | 706 git('fetch', 'origin', cwd=deps_name) |
780 force_revision(deps_name, revision) | 707 force_revision(deps_name, revision) |
781 | 708 |
782 | 709 |
783 def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only, | 710 def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only, |
784 patch_root, issue, patchset, rietveld_server, | 711 patch_root, issue, patchset, rietveld_server, gerrit_repo, |
785 gerrit_repo, gerrit_ref, gerrit_rebase_patch_ref, | 712 gerrit_ref, gerrit_rebase_patch_ref, revision_mapping, |
786 revision_mapping, apply_issue_email_file, | 713 apply_issue_email_file, apply_issue_key_file, shallow, refs, |
787 apply_issue_key_file, gyp_env, shallow, runhooks, | 714 git_cache_dir, gerrit_reset): |
788 refs, git_cache_dir, gerrit_reset): | |
789 # Get a checkout of each solution, without DEPS or hooks. | 715 # Get a checkout of each solution, without DEPS or hooks. |
790 # Calling git directly because there is no way to run Gclient without | 716 # Calling git directly because there is no way to run Gclient without |
791 # invoking DEPS. | 717 # invoking DEPS. |
792 print 'Fetching Git checkout' | 718 print 'Fetching Git checkout' |
793 | 719 |
794 git_ref = git_checkout(solutions, revisions, shallow, refs, git_cache_dir) | 720 git_ref = git_checkout(solutions, revisions, shallow, refs, git_cache_dir) |
795 | 721 |
796 print '===Processing patch solutions===' | 722 print '===Processing patch solutions===' |
797 already_patched = [] | 723 already_patched = [] |
798 patch_root = patch_root or '' | 724 patch_root = patch_root or '' |
(...skipping 136 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
935 parse.add_option('--output_manifest', action='store_true', | 861 parse.add_option('--output_manifest', action='store_true', |
936 help=('Add manifest json to the json output.')) | 862 help=('Add manifest json to the json output.')) |
937 parse.add_option('--slave_name', default=socket.getfqdn().split('.')[0], | 863 parse.add_option('--slave_name', default=socket.getfqdn().split('.')[0], |
938 help='Hostname of the current machine, ' | 864 help='Hostname of the current machine, ' |
939 'used for determining whether or not to activate.') | 865 'used for determining whether or not to activate.') |
940 parse.add_option('--build_dir', default=os.getcwd()) | 866 parse.add_option('--build_dir', default=os.getcwd()) |
941 parse.add_option('--flag_file', default=path.join(os.getcwd(), | 867 parse.add_option('--flag_file', default=path.join(os.getcwd(), |
942 'update.flag')) | 868 'update.flag')) |
943 parse.add_option('--shallow', action='store_true', | 869 parse.add_option('--shallow', action='store_true', |
944 help='Use shallow clones for cache repositories.') | 870 help='Use shallow clones for cache repositories.') |
945 parse.add_option('--gyp_env', action='append', default=[], | |
946 help='Environment variables to pass into gclient runhooks.') | |
947 parse.add_option('--clobber', action='store_true', | 871 parse.add_option('--clobber', action='store_true', |
948 help='Delete checkout first, always') | 872 help='Delete checkout first, always') |
949 parse.add_option('--bot_update_clobber', action='store_true', dest='clobber', | 873 parse.add_option('--bot_update_clobber', action='store_true', dest='clobber', |
950 help='(synonym for --clobber)') | 874 help='(synonym for --clobber)') |
951 parse.add_option('-o', '--output_json', | 875 parse.add_option('-o', '--output_json', |
952 help='Output JSON information into a specified file') | 876 help='Output JSON information into a specified file') |
953 parse.add_option('--no_shallow', action='store_true', | 877 parse.add_option('--no_shallow', action='store_true', |
954 help='Bypass disk detection and never shallow clone. ' | 878 help='Bypass disk detection and never shallow clone. ' |
955 'Does not override the --shallow flag') | 879 'Does not override the --shallow flag') |
956 parse.add_option('--no_runhooks', action='store_true', | |
957 help='Do not run hooks on official builder.') | |
958 parse.add_option('--refs', action='append', | 880 parse.add_option('--refs', action='append', |
959 help='Also fetch this refspec for the main solution(s). ' | 881 help='Also fetch this refspec for the main solution(s). ' |
960 'Eg. +refs/branch-heads/*') | 882 'Eg. +refs/branch-heads/*') |
961 parse.add_option('--with_branch_heads', action='store_true', | 883 parse.add_option('--with_branch_heads', action='store_true', |
962 help='Always pass --with_branch_heads to gclient. This ' | 884 help='Always pass --with_branch_heads to gclient. This ' |
963 'does the same thing as --refs +refs/branch-heads/*') | 885 'does the same thing as --refs +refs/branch-heads/*') |
964 parse.add_option('--git-cache-dir', default=path.join(SLAVE_DIR, 'cache_dir'), | 886 parse.add_option('--git-cache-dir', help='Path to git cache directory.') |
965 help='Path to git cache directory.') | |
966 | 887 |
967 | 888 |
968 options, args = parse.parse_args() | 889 options, args = parse.parse_args() |
969 | 890 |
| 891 if not options.git_cache_dir: |
| 892 parse.error('--git-cache-dir is required') |
| 893 |
970 if not options.refs: | 894 if not options.refs: |
971 options.refs = [] | 895 options.refs = [] |
972 | 896 |
973 if options.with_branch_heads: | 897 if options.with_branch_heads: |
974 options.refs.append(BRANCH_HEADS_REFSPEC) | 898 options.refs.append(BRANCH_HEADS_REFSPEC) |
975 del options.with_branch_heads | 899 del options.with_branch_heads |
976 | 900 |
977 try: | 901 try: |
978 if options.revision_mapping_file: | 902 if options.revision_mapping_file: |
979 if options.revision_mapping: | 903 if options.revision_mapping: |
(...skipping 77 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1057 issue=options.issue, | 981 issue=options.issue, |
1058 patchset=options.patchset, | 982 patchset=options.patchset, |
1059 rietveld_server=options.rietveld_server, | 983 rietveld_server=options.rietveld_server, |
1060 gerrit_repo=options.gerrit_repo, | 984 gerrit_repo=options.gerrit_repo, |
1061 gerrit_ref=options.gerrit_ref, | 985 gerrit_ref=options.gerrit_ref, |
1062 gerrit_rebase_patch_ref=not options.gerrit_no_rebase_patch_ref, | 986 gerrit_rebase_patch_ref=not options.gerrit_no_rebase_patch_ref, |
1063 revision_mapping=options.revision_mapping, | 987 revision_mapping=options.revision_mapping, |
1064 apply_issue_email_file=options.apply_issue_email_file, | 988 apply_issue_email_file=options.apply_issue_email_file, |
1065 apply_issue_key_file=options.apply_issue_key_file, | 989 apply_issue_key_file=options.apply_issue_key_file, |
1066 | 990 |
1067 # For official builders. | |
1068 gyp_env=options.gyp_env, | |
1069 runhooks=not options.no_runhooks, | |
1070 | |
1071 # Finally, extra configurations such as shallowness of the clone. | 991 # Finally, extra configurations such as shallowness of the clone. |
1072 shallow=options.shallow, | 992 shallow=options.shallow, |
1073 refs=options.refs, | 993 refs=options.refs, |
1074 git_cache_dir=options.git_cache_dir, | 994 git_cache_dir=options.git_cache_dir, |
1075 gerrit_reset=not options.gerrit_no_reset) | 995 gerrit_reset=not options.gerrit_no_reset) |
1076 gclient_output = ensure_checkout(**checkout_parameters) | 996 gclient_output = ensure_checkout(**checkout_parameters) |
1077 except GclientSyncFailed: | 997 except GclientSyncFailed: |
1078 print 'We failed gclient sync, lets delete the checkout and retry.' | 998 print 'We failed gclient sync, lets delete the checkout and retry.' |
1079 ensure_no_checkout(dir_names) | 999 ensure_no_checkout(dir_names) |
1080 gclient_output = ensure_checkout(**checkout_parameters) | 1000 gclient_output = ensure_checkout(**checkout_parameters) |
(...skipping 49 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1130 else: | 1050 else: |
1131 # If we're not on recipes, tell annotator about our got_revisions. | 1051 # If we're not on recipes, tell annotator about our got_revisions. |
1132 emit_properties(got_revisions) | 1052 emit_properties(got_revisions) |
1133 | 1053 |
1134 | 1054 |
1135 def print_debug_info(): | 1055 def print_debug_info(): |
1136 print "Debugging info:" | 1056 print "Debugging info:" |
1137 debug_params = { | 1057 debug_params = { |
1138 'CURRENT_DIR': CURRENT_DIR, | 1058 'CURRENT_DIR': CURRENT_DIR, |
1139 'BUILDER_DIR': BUILDER_DIR, | 1059 'BUILDER_DIR': BUILDER_DIR, |
1140 'SLAVE_DIR': SLAVE_DIR, | |
1141 'THIS_DIR': THIS_DIR, | 1060 'THIS_DIR': THIS_DIR, |
1142 'SCRIPTS_DIR': SCRIPTS_DIR, | |
1143 'BUILD_DIR': BUILD_DIR, | |
1144 'ROOT_DIR': ROOT_DIR, | |
1145 'DEPOT_TOOLS_DIR': DEPOT_TOOLS_DIR, | 1061 'DEPOT_TOOLS_DIR': DEPOT_TOOLS_DIR, |
1146 } | 1062 } |
1147 for k, v in sorted(debug_params.iteritems()): | 1063 for k, v in sorted(debug_params.iteritems()): |
1148 print "%s: %r" % (k, v) | 1064 print "%s: %r" % (k, v) |
1149 | 1065 |
1150 | 1066 |
1151 def main(): | 1067 def main(): |
1152 # Get inputs. | 1068 # Get inputs. |
1153 options, _ = parse_args() | 1069 options, _ = parse_args() |
1154 | 1070 |
(...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1190 except Exception: | 1106 except Exception: |
1191 # Unexpected failure. | 1107 # Unexpected failure. |
1192 emit_flag(options.flag_file) | 1108 emit_flag(options.flag_file) |
1193 raise | 1109 raise |
1194 else: | 1110 else: |
1195 emit_flag(options.flag_file) | 1111 emit_flag(options.flag_file) |
1196 | 1112 |
1197 | 1113 |
1198 if __name__ == '__main__': | 1114 if __name__ == '__main__': |
1199 sys.exit(main()) | 1115 sys.exit(main()) |
OLD | NEW |