OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright 2014 The Chromium Authors. All rights reserved. | 2 # Copyright 2014 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 # TODO(hinoka): Use logging. | 6 # TODO(hinoka): Use logging. |
7 | 7 |
8 import cStringIO | 8 import cStringIO |
9 import codecs | 9 import codecs |
10 import collections | 10 import collections |
(...skipping 55 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
66 '..', # ROOT_DIR | 66 '..', # ROOT_DIR |
67 'build', | 67 'build', |
68 'scripts'), | 68 'scripts'), |
69 path.join(SLAVE_DIR, '..', 'build', 'scripts'), | 69 path.join(SLAVE_DIR, '..', 'build', 'scripts'), |
70 ], default=path.dirname(THIS_DIR)) | 70 ], default=path.dirname(THIS_DIR)) |
71 BUILD_DIR = path.dirname(SCRIPTS_DIR) | 71 BUILD_DIR = path.dirname(SCRIPTS_DIR) |
72 ROOT_DIR = path.dirname(BUILD_DIR) | 72 ROOT_DIR = path.dirname(BUILD_DIR) |
73 | 73 |
74 DEPOT_TOOLS_DIR = path.abspath(path.join(THIS_DIR, '..', '..', '..')) | 74 DEPOT_TOOLS_DIR = path.abspath(path.join(THIS_DIR, '..', '..', '..')) |
75 | 75 |
76 BUILD_INTERNAL_DIR = check_dir( | |
77 'build_internal', [ | |
78 path.join(ROOT_DIR, 'build_internal'), | |
79 path.join(ROOT_DIR, # .recipe_deps | |
80 path.pardir, # slave | |
81 path.pardir, # scripts | |
82 path.pardir), # build_internal | |
83 ]) | |
84 | |
85 | |
86 CHROMIUM_GIT_HOST = 'https://chromium.googlesource.com' | 76 CHROMIUM_GIT_HOST = 'https://chromium.googlesource.com' |
87 CHROMIUM_SRC_URL = CHROMIUM_GIT_HOST + '/chromium/src.git' | 77 CHROMIUM_SRC_URL = CHROMIUM_GIT_HOST + '/chromium/src.git' |
88 | 78 |
| 79 RECOGNIZED_PATHS = { |
| 80 '/chrome/trunk/src': |
| 81 CHROMIUM_SRC_URL, |
| 82 '/chrome/trunk/src/tools/cros.DEPS': |
| 83 CHROMIUM_GIT_HOST + '/chromium/src/tools/cros.DEPS.git', |
| 84 '/chrome-internal/trunk/src-internal': |
| 85 'https://chrome-internal.googlesource.com/chrome/src-internal.git', |
| 86 } |
| 87 |
89 # Official builds use buildspecs, so this is a special case. | 88 # Official builds use buildspecs, so this is a special case. |
90 BUILDSPEC_TYPE = collections.namedtuple('buildspec', | 89 BUILDSPEC_TYPE = collections.namedtuple('buildspec', |
91 ('container', 'version')) | 90 ('container', 'version')) |
92 BUILDSPEC_RE = (r'^/chrome-internal/trunk/tools/buildspec/' | 91 BUILDSPEC_RE = (r'^/chrome-internal/trunk/tools/buildspec/' |
93 '(build|branches|releases)/(.+)$') | 92 '(build|branches|releases)/(.+)$') |
94 GIT_BUILDSPEC_PATH = ('https://chrome-internal.googlesource.com/chrome/tools/' | 93 GIT_BUILDSPEC_PATH = ('https://chrome-internal.googlesource.com/chrome/tools/' |
95 'buildspec') | 94 'buildspec') |
96 BRANCH_HEADS_REFSPEC = '+refs/branch-heads/*' | 95 BRANCH_HEADS_REFSPEC = '+refs/branch-heads/*' |
97 | 96 |
98 BUILDSPEC_COMMIT_RE = ( | 97 BUILDSPEC_COMMIT_RE = ( |
(...skipping 62 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
161 'src/tools/swarm_client/': 'got_swarm_client_revision', | 160 'src/tools/swarm_client/': 'got_swarm_client_revision', |
162 'src/tools/swarming_client/': 'got_swarming_client_revision', | 161 'src/tools/swarming_client/': 'got_swarming_client_revision', |
163 'src/third_party/WebKit/': 'got_webkit_revision', | 162 'src/third_party/WebKit/': 'got_webkit_revision', |
164 'src/third_party/webrtc/': 'got_webrtc_revision', | 163 'src/third_party/webrtc/': 'got_webrtc_revision', |
165 'src/v8/': 'got_v8_revision', | 164 'src/v8/': 'got_v8_revision', |
166 } | 165 } |
167 } | 166 } |
168 | 167 |
169 | 168 |
170 BOT_UPDATE_MESSAGE = """ | 169 BOT_UPDATE_MESSAGE = """ |
171 What is the "Bot Update" step? | 170 Bot Update Debugging information: |
172 ============================== | |
173 | |
174 This step ensures that the source checkout on the bot (e.g. Chromium's src/ and | |
175 its dependencies) is checked out in a consistent state. This means that all of | |
176 the necessary repositories are checked out, no extra repositories are checked | |
177 out, and no locally modified files are present. | |
178 | |
179 These actions used to be taken care of by the "gclient revert" and "update" | |
180 steps. However, those steps are known to be buggy and occasionally flaky. This | |
181 step has two main advantages over them: | |
182 * it only operates in Git, so the logic can be clearer and cleaner; and | |
183 * it is a slave-side script, so its behavior can be modified without | |
184 restarting the master. | |
185 | |
186 Why Git, you ask? Because that is the direction that the Chromium project is | |
187 heading. This step is an integral part of the transition from using the SVN repo | |
188 at chrome/trunk/src to using the Git repo src.git. Please pardon the dust while | |
189 we fully convert everything to Git. This message will get out of your way | |
190 eventually, and the waterfall will be a happier place because of it. | |
191 | |
192 This step can be activated or deactivated independently on every builder on | |
193 every master. When it is active, the "gclient revert" and "update" steps become | |
194 no-ops. When it is inactive, it prints this message, cleans up after itself, and | |
195 lets everything else continue as though nothing has changed. Eventually, when | |
196 everything is stable enough, this step will replace them entirely. | |
197 | |
198 Debugging information: | |
199 (master/builder/slave may be unspecified on recipes) | 171 (master/builder/slave may be unspecified on recipes) |
200 master: %(master)s | |
201 builder: %(builder)s | |
202 slave: %(slave)s | |
203 forced by recipes: %(recipe)s | |
204 CURRENT_DIR: %(CURRENT_DIR)s | 172 CURRENT_DIR: %(CURRENT_DIR)s |
205 BUILDER_DIR: %(BUILDER_DIR)s | 173 BUILDER_DIR: %(BUILDER_DIR)s |
206 SLAVE_DIR: %(SLAVE_DIR)s | 174 SLAVE_DIR: %(SLAVE_DIR)s |
207 THIS_DIR: %(THIS_DIR)s | 175 THIS_DIR: %(THIS_DIR)s |
208 SCRIPTS_DIR: %(SCRIPTS_DIR)s | 176 SCRIPTS_DIR: %(SCRIPTS_DIR)s |
209 BUILD_DIR: %(BUILD_DIR)s | 177 BUILD_DIR: %(BUILD_DIR)s |
210 ROOT_DIR: %(ROOT_DIR)s | 178 ROOT_DIR: %(ROOT_DIR)s |
211 DEPOT_TOOLS_DIR: %(DEPOT_TOOLS_DIR)s | 179 DEPOT_TOOLS_DIR: %(DEPOT_TOOLS_DIR)s""" |
212 bot_update.py is:""" | |
213 | |
214 ACTIVATED_MESSAGE = """ACTIVE. | |
215 The bot will perform a Git checkout in this step. | |
216 The "gclient revert" and "update" steps are no-ops. | |
217 | |
218 """ | |
219 | |
220 NOT_ACTIVATED_MESSAGE = """INACTIVE. | |
221 This step does nothing. You actually want to look at the "update" step. | |
222 | |
223 """ | |
224 | |
225 | 180 |
226 GCLIENT_TEMPLATE = """solutions = %(solutions)s | 181 GCLIENT_TEMPLATE = """solutions = %(solutions)s |
227 | 182 |
228 cache_dir = r%(cache_dir)s | 183 cache_dir = r%(cache_dir)s |
229 %(target_os)s | 184 %(target_os)s |
230 %(target_os_only)s | 185 %(target_os_only)s |
231 """ | 186 """ |
232 | 187 |
233 | 188 |
234 internal_data = {} | |
235 if BUILD_INTERNAL_DIR: | |
236 local_vars = {} | |
237 try: | |
238 execfile(os.path.join( | |
239 BUILD_INTERNAL_DIR, 'scripts', 'slave', 'bot_update_cfg.py'), | |
240 local_vars) | |
241 except Exception: | |
242 # Same as if BUILD_INTERNAL_DIR didn't exist in the first place. | |
243 print 'Warning: unable to read internal configuration file.' | |
244 print 'If this is an internal bot, this step may be erroneously inactive.' | |
245 internal_data = local_vars | |
246 | |
247 RECOGNIZED_PATHS = { | |
248 # If SVN path matches key, the entire URL is rewritten to the Git url. | |
249 '/chrome/trunk/src': | |
250 CHROMIUM_SRC_URL, | |
251 '/chrome/trunk/src/tools/cros.DEPS': | |
252 CHROMIUM_GIT_HOST + '/chromium/src/tools/cros.DEPS.git', | |
253 } | |
254 RECOGNIZED_PATHS.update(internal_data.get('RECOGNIZED_PATHS', {})) | |
255 | |
256 ENABLED_MASTERS = [ | |
257 'bot_update.always_on', | |
258 'chromium.android', | |
259 'chromium.angle', | |
260 'chromium.chrome', | |
261 'chromium.chromedriver', | |
262 'chromium.chromiumos', | |
263 'chromium', | |
264 'chromium.fyi', | |
265 'chromium.goma', | |
266 'chromium.gpu', | |
267 'chromium.gpu.fyi', | |
268 'chromium.infra', | |
269 'chromium.infra.cron', | |
270 'chromium.linux', | |
271 'chromium.lkgr', | |
272 'chromium.mac', | |
273 'chromium.memory', | |
274 'chromium.memory.fyi', | |
275 'chromium.perf', | |
276 'chromium.perf.fyi', | |
277 'chromium.swarm', | |
278 'chromium.webkit', | |
279 'chromium.webrtc', | |
280 'chromium.webrtc.fyi', | |
281 'chromium.win', | |
282 'client.catapult', | |
283 'client.drmemory', | |
284 'client.mojo', | |
285 'client.nacl', | |
286 'client.nacl.ports', | |
287 'client.nacl.sdk', | |
288 'client.nacl.toolchain', | |
289 'client.pdfium', | |
290 'client.skia', | |
291 'client.skia.fyi', | |
292 'client.v8', | |
293 'client.v8.branches', | |
294 'client.v8.fyi', | |
295 'client.webrtc', | |
296 'client.webrtc.fyi', | |
297 'tryserver.blink', | |
298 'tryserver.client.catapult', | |
299 'tryserver.client.mojo', | |
300 'tryserver.chromium.android', | |
301 'tryserver.chromium.angle', | |
302 'tryserver.chromium.linux', | |
303 'tryserver.chromium.mac', | |
304 'tryserver.chromium.perf', | |
305 'tryserver.chromium.win', | |
306 'tryserver.infra', | |
307 'tryserver.nacl', | |
308 'tryserver.v8', | |
309 'tryserver.webrtc', | |
310 ] | |
311 ENABLED_MASTERS += internal_data.get('ENABLED_MASTERS', []) | |
312 | |
313 ENABLED_BUILDERS = { | |
314 'client.dart.fyi': [ | |
315 'v8-linux-release', | |
316 'v8-mac-release', | |
317 'v8-win-release', | |
318 ], | |
319 'client.dynamorio': [ | |
320 'linux-v8-dr', | |
321 ], | |
322 } | |
323 ENABLED_BUILDERS.update(internal_data.get('ENABLED_BUILDERS', {})) | |
324 | |
325 ENABLED_SLAVES = {} | |
326 ENABLED_SLAVES.update(internal_data.get('ENABLED_SLAVES', {})) | |
327 | |
328 # Disabled filters get run AFTER enabled filters, so for example if a builder | |
329 # config is enabled, but a bot on that builder is disabled, that bot will | |
330 # be disabled. | |
331 DISABLED_BUILDERS = {} | |
332 DISABLED_BUILDERS.update(internal_data.get('DISABLED_BUILDERS', {})) | |
333 | |
334 DISABLED_SLAVES = {} | |
335 DISABLED_SLAVES.update(internal_data.get('DISABLED_SLAVES', {})) | |
336 | |
337 # These masters work only in Git, meaning for got_revision, always output | |
338 # a git hash rather than a SVN rev. | |
339 GIT_MASTERS = [ | |
340 'client.v8', | |
341 'client.v8.branches', | |
342 'tryserver.v8', | |
343 ] | |
344 GIT_MASTERS += internal_data.get('GIT_MASTERS', []) | |
345 | |
346 | |
347 # How many times to try before giving up. | 189 # How many times to try before giving up. |
348 ATTEMPTS = 5 | 190 ATTEMPTS = 5 |
349 | 191 |
350 # Find deps2git | |
351 DEPS2GIT_DIR_PATH = path.join(SCRIPTS_DIR, 'tools', 'deps2git') | |
352 DEPS2GIT_PATH = path.join(DEPS2GIT_DIR_PATH, 'deps2git.py') | |
353 S2G_INTERNAL_PATH = path.join(SCRIPTS_DIR, 'tools', 'deps2git_internal', | |
354 'svn_to_git_internal.py') | |
355 GIT_CACHE_PATH = path.join(DEPOT_TOOLS_DIR, 'git_cache.py') | 192 GIT_CACHE_PATH = path.join(DEPOT_TOOLS_DIR, 'git_cache.py') |
356 | 193 |
357 # Find the patch tool. | 194 # Find the patch tool. |
358 if sys.platform.startswith('win'): | 195 if sys.platform.startswith('win'): |
359 if not BUILD_INTERNAL_DIR: | 196 PATCH_TOOL = path.join(THIS_DIR, 'patch.EXE') |
360 print 'Warning: could not find patch tool because there is no ' | |
361 print 'build_internal present.' | |
362 PATCH_TOOL = None | |
363 else: | |
364 PATCH_TOOL = path.join(BUILD_INTERNAL_DIR, 'tools', 'patch.EXE') | |
365 else: | 197 else: |
366 PATCH_TOOL = '/usr/bin/patch' | 198 PATCH_TOOL = '/usr/bin/patch' |
367 | 199 |
368 # If there is less than 100GB of disk space on the system, then we do | 200 # If there is less than 100GB of disk space on the system, then we do |
369 # a shallow checkout. | 201 # a shallow checkout. |
370 SHALLOW_CLONE_THRESHOLD = 100 * 1024 * 1024 * 1024 | 202 SHALLOW_CLONE_THRESHOLD = 100 * 1024 * 1024 * 1024 |
371 | 203 |
372 | 204 |
373 class SubprocessFailed(Exception): | 205 class SubprocessFailed(Exception): |
374 def __init__(self, message, code, output): | 206 def __init__(self, message, code, output): |
(...skipping 11 matching lines...) Expand all Loading... |
386 | 218 |
387 | 219 |
388 class SVNRevisionNotFound(Exception): | 220 class SVNRevisionNotFound(Exception): |
389 pass | 221 pass |
390 | 222 |
391 | 223 |
392 class InvalidDiff(Exception): | 224 class InvalidDiff(Exception): |
393 pass | 225 pass |
394 | 226 |
395 | 227 |
396 class Inactive(Exception): | |
397 """Not really an exception, just used to exit early cleanly.""" | |
398 pass | |
399 | |
400 | |
401 RETRY = object() | 228 RETRY = object() |
402 OK = object() | 229 OK = object() |
403 FAIL = object() | 230 FAIL = object() |
404 | 231 |
405 | 232 |
406 class PsPrinter(object): | 233 class PsPrinter(object): |
407 def __init__(self, interval=300): | 234 def __init__(self, interval=300): |
408 self.interval = interval | 235 self.interval = interval |
409 self.active = sys.platform.startswith('linux2') | 236 self.active = sys.platform.startswith('linux2') |
410 self.thread = None | 237 self.thread = None |
(...skipping 108 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
519 | 346 |
520 def get_gclient_spec(solutions, target_os, target_os_only, git_cache_dir): | 347 def get_gclient_spec(solutions, target_os, target_os_only, git_cache_dir): |
521 return GCLIENT_TEMPLATE % { | 348 return GCLIENT_TEMPLATE % { |
522 'solutions': pprint.pformat(solutions, indent=4), | 349 'solutions': pprint.pformat(solutions, indent=4), |
523 'cache_dir': '"%s"' % git_cache_dir, | 350 'cache_dir': '"%s"' % git_cache_dir, |
524 'target_os': ('\ntarget_os=%s' % target_os) if target_os else '', | 351 'target_os': ('\ntarget_os=%s' % target_os) if target_os else '', |
525 'target_os_only': '\ntarget_os_only=%s' % target_os_only | 352 'target_os_only': '\ntarget_os_only=%s' % target_os_only |
526 } | 353 } |
527 | 354 |
528 | 355 |
529 def check_enabled(master, builder, slave): | |
530 if master in ENABLED_MASTERS: | |
531 return True | |
532 builder_list = ENABLED_BUILDERS.get(master) | |
533 if builder_list and builder in builder_list: | |
534 return True | |
535 slave_list = ENABLED_SLAVES.get(master) | |
536 if slave_list and slave in slave_list: | |
537 return True | |
538 return False | |
539 | |
540 | |
541 def check_disabled(master, builder, slave): | |
542 """Returns True if disabled, False if not disabled.""" | |
543 builder_list = DISABLED_BUILDERS.get(master) | |
544 if builder_list and builder in builder_list: | |
545 return True | |
546 slave_list = DISABLED_SLAVES.get(master) | |
547 if slave_list and slave in slave_list: | |
548 return True | |
549 return False | |
550 | |
551 | |
552 def check_valid_host(master, builder, slave): | |
553 return (check_enabled(master, builder, slave) | |
554 and not check_disabled(master, builder, slave)) | |
555 | |
556 | |
557 def maybe_ignore_revision(revision, buildspec): | 356 def maybe_ignore_revision(revision, buildspec): |
558 """Handle builders that don't care what buildbot tells them to build. | 357 """Handle builders that don't care what buildbot tells them to build. |
559 | 358 |
560 This is especially the case with branch builders that build from buildspecs | 359 This is especially the case with branch builders that build from buildspecs |
561 and/or trigger off multiple repositories, where the --revision passed in has | 360 and/or trigger off multiple repositories, where the --revision passed in has |
562 nothing to do with the solution being built. Clearing the revision in this | 361 nothing to do with the solution being built. Clearing the revision in this |
563 case causes bot_update to use HEAD rather that trying to checkout an | 362 case causes bot_update to use HEAD rather that trying to checkout an |
564 inappropriate version of the solution. | 363 inappropriate version of the solution. |
565 """ | 364 """ |
566 if buildspec and buildspec.container == 'branches': | 365 if buildspec and buildspec.container == 'branches': |
(...skipping 207 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
774 footers[m.group(1)] = m.group(2).strip() | 573 footers[m.group(1)] = m.group(2).strip() |
775 return footers | 574 return footers |
776 | 575 |
777 | 576 |
778 def get_commit_message_footer(message, key): | 577 def get_commit_message_footer(message, key): |
779 """Returns: (str/None) The footer value for 'key', or None if none was found. | 578 """Returns: (str/None) The footer value for 'key', or None if none was found. |
780 """ | 579 """ |
781 return get_commit_message_footer_map(message).get(key) | 580 return get_commit_message_footer_map(message).get(key) |
782 | 581 |
783 | 582 |
784 def get_svn_rev(git_hash, dir_name): | |
785 log = git('log', '-1', git_hash, cwd=dir_name) | |
786 git_svn_id = get_commit_message_footer(log, GIT_SVN_ID_FOOTER_KEY) | |
787 if not git_svn_id: | |
788 return None | |
789 m = GIT_SVN_ID_RE.match(git_svn_id) | |
790 if not m: | |
791 return None | |
792 return int(m.group(2)) | |
793 | |
794 | |
795 def get_git_hash(revision, branch, sln_dir): | 583 def get_git_hash(revision, branch, sln_dir): |
796 """We want to search for the SVN revision on the git-svn branch. | 584 """We want to search for the SVN revision on the git-svn branch. |
797 | 585 |
798 Note that git will search backwards from origin/master. | 586 Note that git will search backwards from origin/master. |
799 """ | 587 """ |
800 match = "^%s: [^ ]*@%s " % (GIT_SVN_ID_FOOTER_KEY, revision) | 588 match = "^%s: [^ ]*@%s " % (GIT_SVN_ID_FOOTER_KEY, revision) |
801 ref = branch if branch.startswith('refs/') else 'origin/%s' % branch | 589 ref = branch if branch.startswith('refs/') else 'origin/%s' % branch |
802 cmd = ['log', '-E', '--grep', match, '--format=%H', '--max-count=1', ref] | 590 cmd = ['log', '-E', '--grep', match, '--format=%H', '--max-count=1', ref] |
803 result = git(*cmd, cwd=sln_dir).strip() | 591 result = git(*cmd, cwd=sln_dir).strip() |
804 if result: | 592 if result: |
805 return result | 593 return result |
806 raise SVNRevisionNotFound('We can\'t resolve svn r%s into a git hash in %s' % | 594 raise SVNRevisionNotFound('We can\'t resolve svn r%s into a git hash in %s' % |
807 (revision, sln_dir)) | 595 (revision, sln_dir)) |
808 | 596 |
809 | 597 |
810 def _last_commit_for_file(filename, repo_base): | |
811 cmd = ['log', '--format=%H', '--max-count=1', '--', filename] | |
812 return git(*cmd, cwd=repo_base).strip() | |
813 | |
814 | |
815 def need_to_run_deps2git(repo_base, deps_file, deps_git_file): | |
816 """Checks to see if we need to run deps2git. | |
817 | |
818 Returns True if there was a DEPS change after the last .DEPS.git update | |
819 or if DEPS has local modifications. | |
820 """ | |
821 # See if DEPS is dirty | |
822 deps_file_status = git( | |
823 'status', '--porcelain', deps_file, cwd=repo_base).strip() | |
824 if deps_file_status and deps_file_status.startswith('M '): | |
825 return True | |
826 | |
827 last_known_deps_ref = _last_commit_for_file(deps_file, repo_base) | |
828 last_known_deps_git_ref = _last_commit_for_file(deps_git_file, repo_base) | |
829 merge_base_ref = git('merge-base', last_known_deps_ref, | |
830 last_known_deps_git_ref, cwd=repo_base).strip() | |
831 | |
832 # If the merge base of the last DEPS and last .DEPS.git file is not | |
833 # equivilent to the hash of the last DEPS file, that means the DEPS file | |
834 # was committed after the last .DEPS.git file. | |
835 return last_known_deps_ref != merge_base_ref | |
836 | |
837 | |
838 def ensure_deps2git(solution, shallow, git_cache_dir): | |
839 repo_base = path.join(os.getcwd(), solution['name']) | |
840 deps_file = path.join(repo_base, 'DEPS') | |
841 deps_git_file = path.join(repo_base, '.DEPS.git') | |
842 if (not git('ls-files', 'DEPS', cwd=repo_base).strip() or | |
843 not git('ls-files', '.DEPS.git', cwd=repo_base).strip()): | |
844 return | |
845 | |
846 print 'Checking if %s is newer than %s' % (deps_file, deps_git_file) | |
847 if not need_to_run_deps2git(repo_base, deps_file, deps_git_file): | |
848 return | |
849 | |
850 print '===DEPS file modified, need to run deps2git===' | |
851 cmd = [sys.executable, DEPS2GIT_PATH, | |
852 '--workspace', os.getcwd(), | |
853 '--cache_dir', git_cache_dir, | |
854 '--deps', deps_file, | |
855 '--out', deps_git_file] | |
856 if 'chrome-internal.googlesource' in solution['url']: | |
857 cmd.extend(['--extra-rules', S2G_INTERNAL_PATH]) | |
858 if shallow: | |
859 cmd.append('--shallow') | |
860 call(*cmd) | |
861 | |
862 | |
863 def emit_log_lines(name, lines): | 598 def emit_log_lines(name, lines): |
864 for line in lines.splitlines(): | 599 for line in lines.splitlines(): |
865 print '@@@STEP_LOG_LINE@%s@%s@@@' % (name, line) | 600 print '@@@STEP_LOG_LINE@%s@%s@@@' % (name, line) |
866 print '@@@STEP_LOG_END@%s@@@' % name | 601 print '@@@STEP_LOG_END@%s@@@' % name |
867 | 602 |
868 | 603 |
869 def emit_properties(properties): | 604 def emit_properties(properties): |
870 for property_name, property_value in sorted(properties.items()): | 605 for property_name, property_value in sorted(properties.items()): |
871 print '@@@SET_BUILD_PROPERTY@%s@"%s"@@@' % (property_name, property_value) | 606 print '@@@SET_BUILD_PROPERTY@%s@"%s"@@@' % (property_name, property_value) |
872 | 607 |
(...skipping 45 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
918 # rev_num is really a svn revision number, convert it into a git hash. | 653 # rev_num is really a svn revision number, convert it into a git hash. |
919 git_ref = get_git_hash(int(revision), branch, folder_name) | 654 git_ref = get_git_hash(int(revision), branch, folder_name) |
920 else: | 655 else: |
921 # rev_num is actually a git hash or ref, we can just use it. | 656 # rev_num is actually a git hash or ref, we can just use it. |
922 git_ref = revision | 657 git_ref = revision |
923 git('checkout', '--force', git_ref, cwd=folder_name) | 658 git('checkout', '--force', git_ref, cwd=folder_name) |
924 else: | 659 else: |
925 ref = branch if branch.startswith('refs/') else 'origin/%s' % branch | 660 ref = branch if branch.startswith('refs/') else 'origin/%s' % branch |
926 git('checkout', '--force', ref, cwd=folder_name) | 661 git('checkout', '--force', ref, cwd=folder_name) |
927 | 662 |
| 663 |
928 def git_checkout(solutions, revisions, shallow, refs, git_cache_dir): | 664 def git_checkout(solutions, revisions, shallow, refs, git_cache_dir): |
929 build_dir = os.getcwd() | 665 build_dir = os.getcwd() |
930 # Before we do anything, break all git_cache locks. | 666 # Before we do anything, break all git_cache locks. |
931 if path.isdir(git_cache_dir): | 667 if path.isdir(git_cache_dir): |
932 git('cache', 'unlock', '-vv', '--force', '--all', | 668 git('cache', 'unlock', '-vv', '--force', '--all', |
933 '--cache-dir', git_cache_dir) | 669 '--cache-dir', git_cache_dir) |
934 for item in os.listdir(git_cache_dir): | 670 for item in os.listdir(git_cache_dir): |
935 filename = os.path.join(git_cache_dir, item) | 671 filename = os.path.join(git_cache_dir, item) |
936 if item.endswith('.lock'): | 672 if item.endswith('.lock'): |
937 raise Exception('%s exists after cache unlock' % filename) | 673 raise Exception('%s exists after cache unlock' % filename) |
(...skipping 40 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
978 # Exited abnormally, theres probably something wrong. | 714 # Exited abnormally, theres probably something wrong. |
979 # Lets wipe the checkout and try again. | 715 # Lets wipe the checkout and try again. |
980 tries_left -= 1 | 716 tries_left -= 1 |
981 if tries_left > 0: | 717 if tries_left > 0: |
982 print 'Something failed: %s.' % str(e) | 718 print 'Something failed: %s.' % str(e) |
983 print 'waiting 5 seconds and trying again...' | 719 print 'waiting 5 seconds and trying again...' |
984 time.sleep(5) | 720 time.sleep(5) |
985 else: | 721 else: |
986 raise | 722 raise |
987 remove(sln_dir) | 723 remove(sln_dir) |
988 except SVNRevisionNotFound: | |
989 tries_left -= 1 | |
990 if tries_left > 0: | |
991 # If we don't have the correct revision, wait and try again. | |
992 print 'We can\'t find revision %s.' % revision | |
993 print 'The svn to git replicator is probably falling behind.' | |
994 print 'waiting 5 seconds and trying again...' | |
995 time.sleep(5) | |
996 else: | |
997 raise | |
998 | 724 |
999 git('clean', '-dff', cwd=sln_dir) | 725 git('clean', '-dff', cwd=sln_dir) |
1000 | 726 |
1001 if first_solution: | 727 if first_solution: |
1002 git_ref = git('log', '--format=%H', '--max-count=1', | 728 git_ref = git('log', '--format=%H', '--max-count=1', |
1003 cwd=sln_dir).strip() | 729 cwd=sln_dir).strip() |
1004 first_solution = False | 730 first_solution = False |
1005 return git_ref | 731 return git_ref |
1006 | 732 |
1007 | 733 |
1008 def _download(url): | |
1009 """Fetch url and return content, with retries for flake.""" | |
1010 for attempt in xrange(ATTEMPTS): | |
1011 try: | |
1012 return urllib2.urlopen(url).read() | |
1013 except Exception: | |
1014 if attempt == ATTEMPTS - 1: | |
1015 raise | |
1016 | |
1017 | |
1018 def parse_diff(diff): | 734 def parse_diff(diff): |
1019 """Takes a unified diff and returns a list of diffed files and their diffs. | 735 """Takes a unified diff and returns a list of diffed files and their diffs. |
1020 | 736 |
1021 The return format is a list of pairs of: | 737 The return format is a list of pairs of: |
1022 (<filename>, <diff contents>) | 738 (<filename>, <diff contents>) |
1023 <diff contents> is inclusive of the diff line. | 739 <diff contents> is inclusive of the diff line. |
1024 """ | 740 """ |
1025 result = [] | 741 result = [] |
1026 current_diff = '' | 742 current_diff = '' |
1027 current_header = None | 743 current_header = None |
(...skipping 189 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1217 # Compose a commit position from 'git-svn' metadata | 933 # Compose a commit position from 'git-svn' metadata |
1218 value = footer_map.get(GIT_SVN_ID_FOOTER_KEY) | 934 value = footer_map.get(GIT_SVN_ID_FOOTER_KEY) |
1219 if value: | 935 if value: |
1220 m = GIT_SVN_ID_RE.match(value) | 936 m = GIT_SVN_ID_RE.match(value) |
1221 if not m: | 937 if not m: |
1222 raise ValueError("Invalid 'git-svn' value: [%s]" % (value,)) | 938 raise ValueError("Invalid 'git-svn' value: [%s]" % (value,)) |
1223 return get_commit_position_for_git_svn(m.group(1), m.group(2)) | 939 return get_commit_position_for_git_svn(m.group(1), m.group(2)) |
1224 return None | 940 return None |
1225 | 941 |
1226 | 942 |
1227 def parse_got_revision(gclient_output, got_revision_mapping, use_svn_revs): | 943 def parse_got_revision(gclient_output, got_revision_mapping): |
1228 """Translate git gclient revision mapping to build properties. | 944 """Translate git gclient revision mapping to build properties.""" |
1229 | |
1230 If use_svn_revs is True, then translate git hashes in the revision mapping | |
1231 to svn revision numbers. | |
1232 """ | |
1233 properties = {} | 945 properties = {} |
1234 solutions_output = { | 946 solutions_output = { |
1235 # Make sure path always ends with a single slash. | 947 # Make sure path always ends with a single slash. |
1236 '%s/' % path.rstrip('/') : solution_output for path, solution_output | 948 '%s/' % path.rstrip('/') : solution_output for path, solution_output |
1237 in gclient_output['solutions'].iteritems() | 949 in gclient_output['solutions'].iteritems() |
1238 } | 950 } |
1239 for dir_name, property_name in got_revision_mapping.iteritems(): | 951 for dir_name, property_name in got_revision_mapping.iteritems(): |
1240 # Make sure dir_name always ends with a single slash. | 952 # Make sure dir_name always ends with a single slash. |
1241 dir_name = '%s/' % dir_name.rstrip('/') | 953 dir_name = '%s/' % dir_name.rstrip('/') |
1242 if dir_name not in solutions_output: | 954 if dir_name not in solutions_output: |
1243 continue | 955 continue |
1244 solution_output = solutions_output[dir_name] | 956 solution_output = solutions_output[dir_name] |
1245 if solution_output.get('scm') is None: | 957 if solution_output.get('scm') is None: |
1246 # This is an ignored DEPS, so the output got_revision should be 'None'. | 958 # This is an ignored DEPS, so the output got_revision should be 'None'. |
1247 git_revision = revision = commit_position = None | 959 git_revision = revision = commit_position = None |
1248 else: | 960 else: |
1249 # Since we are using .DEPS.git, everything had better be git. | 961 # Since we are using .DEPS.git, everything had better be git. |
1250 assert solution_output.get('scm') == 'git' | 962 assert solution_output.get('scm') == 'git' |
1251 git_revision = git('rev-parse', 'HEAD', cwd=dir_name).strip() | 963 git_revision = git('rev-parse', 'HEAD', cwd=dir_name).strip() |
1252 if use_svn_revs: | 964 revision = git_revision |
1253 revision = get_svn_rev(git_revision, dir_name) | |
1254 if not revision: | |
1255 revision = git_revision | |
1256 else: | |
1257 revision = git_revision | |
1258 commit_position = get_commit_position(dir_name) | 965 commit_position = get_commit_position(dir_name) |
1259 | 966 |
1260 properties[property_name] = revision | 967 properties[property_name] = revision |
1261 if revision != git_revision: | 968 if revision != git_revision: |
1262 properties['%s_git' % property_name] = git_revision | 969 properties['%s_git' % property_name] = git_revision |
1263 if commit_position: | 970 if commit_position: |
1264 properties['%s_cp' % property_name] = commit_position | 971 properties['%s_cp' % property_name] = commit_position |
1265 | 972 |
1266 return properties | 973 return properties |
1267 | 974 |
(...skipping 11 matching lines...) Expand all Loading... |
1279 def ensure_deps_revisions(deps_url_mapping, solutions, revisions): | 986 def ensure_deps_revisions(deps_url_mapping, solutions, revisions): |
1280 """Ensure correct DEPS revisions, ignores solutions.""" | 987 """Ensure correct DEPS revisions, ignores solutions.""" |
1281 for deps_name, deps_data in sorted(deps_url_mapping.items()): | 988 for deps_name, deps_data in sorted(deps_url_mapping.items()): |
1282 if deps_name.strip('/') in solutions: | 989 if deps_name.strip('/') in solutions: |
1283 # This has already been forced to the correct solution by git_checkout(). | 990 # This has already been forced to the correct solution by git_checkout(). |
1284 continue | 991 continue |
1285 revision = get_target_revision(deps_name, deps_data.get('url', None), | 992 revision = get_target_revision(deps_name, deps_data.get('url', None), |
1286 revisions) | 993 revisions) |
1287 if not revision: | 994 if not revision: |
1288 continue | 995 continue |
1289 # TODO(hinoka): Catch SVNRevisionNotFound error maybe? | |
1290 git('fetch', 'origin', cwd=deps_name) | 996 git('fetch', 'origin', cwd=deps_name) |
1291 force_revision(deps_name, revision) | 997 force_revision(deps_name, revision) |
1292 | 998 |
1293 | 999 |
1294 def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only, | 1000 def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only, |
1295 patch_root, issue, patchset, patch_url, rietveld_server, | 1001 patch_root, issue, patchset, patch_url, rietveld_server, |
1296 gerrit_repo, gerrit_ref, revision_mapping, | 1002 gerrit_repo, gerrit_ref, revision_mapping, |
1297 apply_issue_email_file, apply_issue_key_file, buildspec, | 1003 apply_issue_email_file, apply_issue_key_file, buildspec, |
1298 gyp_env, shallow, runhooks, refs, git_cache_dir): | 1004 gyp_env, shallow, runhooks, refs, git_cache_dir): |
1299 # Get a checkout of each solution, without DEPS or hooks. | 1005 # Get a checkout of each solution, without DEPS or hooks. |
(...skipping 16 matching lines...) Expand all Loading... |
1316 target = '/'.join([relative_root, 'DEPS']).lstrip('/') | 1022 target = '/'.join([relative_root, 'DEPS']).lstrip('/') |
1317 if patches: | 1023 if patches: |
1318 apply_svn_patch(patch_root, patches, whitelist=[target]) | 1024 apply_svn_patch(patch_root, patches, whitelist=[target]) |
1319 already_patched.append(target) | 1025 already_patched.append(target) |
1320 elif issue: | 1026 elif issue: |
1321 apply_rietveld_issue(issue, patchset, patch_root, rietveld_server, | 1027 apply_rietveld_issue(issue, patchset, patch_root, rietveld_server, |
1322 revision_mapping, git_ref, apply_issue_email_file, | 1028 revision_mapping, git_ref, apply_issue_email_file, |
1323 apply_issue_key_file, whitelist=[target]) | 1029 apply_issue_key_file, whitelist=[target]) |
1324 already_patched.append(target) | 1030 already_patched.append(target) |
1325 | 1031 |
1326 if not buildspec: | |
1327 # Run deps2git if there is a DEPS change after the last .DEPS.git commit. | |
1328 for solution in solutions: | |
1329 ensure_deps2git(solution, shallow, git_cache_dir) | |
1330 | |
1331 # Ensure our build/ directory is set up with the correct .gclient file. | 1032 # Ensure our build/ directory is set up with the correct .gclient file. |
1332 gclient_configure(solutions, target_os, target_os_only, git_cache_dir) | 1033 gclient_configure(solutions, target_os, target_os_only, git_cache_dir) |
1333 | 1034 |
1334 # Let gclient do the DEPS syncing. | 1035 # Let gclient do the DEPS syncing. |
1335 # The branch-head refspec is a special case because its possible Chrome | 1036 # The branch-head refspec is a special case because its possible Chrome |
1336 # src, which contains the branch-head refspecs, is DEPSed in. | 1037 # src, which contains the branch-head refspecs, is DEPSed in. |
1337 gclient_output = gclient_sync(buildspec or BRANCH_HEADS_REFSPEC in refs, | 1038 gclient_output = gclient_sync(buildspec or BRANCH_HEADS_REFSPEC in refs, |
1338 shallow) | 1039 shallow) |
1339 | 1040 |
1340 # Now that gclient_sync has finished, we should revert any .DEPS.git so that | 1041 # Now that gclient_sync has finished, we should revert any .DEPS.git so that |
(...skipping 83 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1424 | 1125 |
1425 parse.add_option('--issue', help='Issue number to patch from.') | 1126 parse.add_option('--issue', help='Issue number to patch from.') |
1426 parse.add_option('--patchset', | 1127 parse.add_option('--patchset', |
1427 help='Patchset from issue to patch from, if applicable.') | 1128 help='Patchset from issue to patch from, if applicable.') |
1428 parse.add_option('--apply_issue_email_file', | 1129 parse.add_option('--apply_issue_email_file', |
1429 help='--email-file option passthrough for apply_patch.py.') | 1130 help='--email-file option passthrough for apply_patch.py.') |
1430 parse.add_option('--apply_issue_key_file', | 1131 parse.add_option('--apply_issue_key_file', |
1431 help='--private-key-file option passthrough for ' | 1132 help='--private-key-file option passthrough for ' |
1432 'apply_patch.py.') | 1133 'apply_patch.py.') |
1433 parse.add_option('--patch_url', help='Optional URL to SVN patch.') | 1134 parse.add_option('--patch_url', help='Optional URL to SVN patch.') |
1434 parse.add_option('--root', dest='patch_root', | |
1435 help='DEPRECATED: Use --patch_root.') | |
1436 parse.add_option('--patch_root', help='Directory to patch on top of.') | 1135 parse.add_option('--patch_root', help='Directory to patch on top of.') |
1437 parse.add_option('--rietveld_server', | 1136 parse.add_option('--rietveld_server', |
1438 default='codereview.chromium.org', | 1137 default='codereview.chromium.org', |
1439 help='Rietveld server.') | 1138 help='Rietveld server.') |
1440 parse.add_option('--gerrit_repo', | 1139 parse.add_option('--gerrit_repo', |
1441 help='Gerrit repository to pull the ref from.') | 1140 help='Gerrit repository to pull the ref from.') |
1442 parse.add_option('--gerrit_ref', help='Gerrit ref to apply.') | 1141 parse.add_option('--gerrit_ref', help='Gerrit ref to apply.') |
1443 parse.add_option('--specs', help='Gcilent spec.') | 1142 parse.add_option('--specs', help='Gcilent spec.') |
1444 parse.add_option('--master', help='Master name.') | |
1445 parse.add_option('-f', '--force', action='store_true', | |
1446 help='Bypass check to see if we want to be run. ' | |
1447 'Should ONLY be used locally or by smart recipes.') | |
1448 parse.add_option('--revision_mapping', | 1143 parse.add_option('--revision_mapping', |
1449 help='{"path/to/repo/": "property_name"}') | 1144 help='{"path/to/repo/": "property_name"}') |
1450 parse.add_option('--revision_mapping_file', | 1145 parse.add_option('--revision_mapping_file', |
1451 help=('Same as revision_mapping, except its a path to a json' | 1146 help=('Same as revision_mapping, except its a path to a json' |
1452 ' file containing that format.')) | 1147 ' file containing that format.')) |
1453 parse.add_option('--revision', action='append', default=[], | 1148 parse.add_option('--revision', action='append', default=[], |
1454 help='Revision to check out. Can be an SVN revision number, ' | 1149 help='Revision to check out. Can be an SVN revision number, ' |
1455 'git hash, or any form of git ref. Can prepend ' | 1150 'git hash, or any form of git ref. Can prepend ' |
1456 'root@<rev> to specify which repository, where root ' | 1151 'root@<rev> to specify which repository, where root ' |
1457 'is either a filesystem path, git https url, or ' | 1152 'is either a filesystem path, git https url, or ' |
1458 'svn url. To specify Tip of Tree, set rev to HEAD.' | 1153 'svn url. To specify Tip of Tree, set rev to HEAD.' |
1459 'To specify a git branch and an SVN rev, <rev> can be ' | 1154 'To specify a git branch and an SVN rev, <rev> can be ' |
1460 'set to <branch>:<revision>.') | 1155 'set to <branch>:<revision>.') |
1461 parse.add_option('--output_manifest', action='store_true', | 1156 parse.add_option('--output_manifest', action='store_true', |
1462 help=('Add manifest json to the json output.')) | 1157 help=('Add manifest json to the json output.')) |
1463 parse.add_option('--slave_name', default=socket.getfqdn().split('.')[0], | |
1464 help='Hostname of the current machine, ' | |
1465 'used for determining whether or not to activate.') | |
1466 parse.add_option('--builder_name', help='Name of the builder, ' | |
1467 'used for determining whether or not to activate.') | |
1468 parse.add_option('--build_dir', default=os.getcwd()) | 1158 parse.add_option('--build_dir', default=os.getcwd()) |
1469 parse.add_option('--flag_file', default=path.join(os.getcwd(), | 1159 parse.add_option('--flag_file', default=path.join(os.getcwd(), |
1470 'update.flag')) | 1160 'update.flag')) |
1471 parse.add_option('--shallow', action='store_true', | 1161 parse.add_option('--shallow', action='store_true', |
1472 help='Use shallow clones for cache repositories.') | 1162 help='Use shallow clones for cache repositories.') |
1473 parse.add_option('--gyp_env', action='append', default=[], | 1163 parse.add_option('--gyp_env', action='append', default=[], |
1474 help='Environment variables to pass into gclient runhooks.') | 1164 help='Environment variables to pass into gclient runhooks.') |
1475 parse.add_option('--clobber', action='store_true', | 1165 parse.add_option('--clobber', action='store_true', |
1476 help='Delete checkout first, always') | 1166 help='Delete checkout first, always') |
1477 parse.add_option('--bot_update_clobber', action='store_true', dest='clobber', | 1167 parse.add_option('--bot_update_clobber', action='store_true', dest='clobber', |
(...skipping 41 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1519 | 1209 |
1520 # Because we print CACHE_DIR out into a .gclient file, and then later run | 1210 # Because we print CACHE_DIR out into a .gclient file, and then later run |
1521 # eval() on it, backslashes need to be escaped, otherwise "E:\b\build" gets | 1211 # eval() on it, backslashes need to be escaped, otherwise "E:\b\build" gets |
1522 # parsed as "E:[\x08][\x08]uild". | 1212 # parsed as "E:[\x08][\x08]uild". |
1523 if sys.platform.startswith('win'): | 1213 if sys.platform.startswith('win'): |
1524 options.git_cache_dir = options.git_cache_dir.replace('\\', '\\\\') | 1214 options.git_cache_dir = options.git_cache_dir.replace('\\', '\\\\') |
1525 | 1215 |
1526 return options, args | 1216 return options, args |
1527 | 1217 |
1528 | 1218 |
1529 def prepare(options, git_slns, active): | 1219 def prepare(options, git_slns): |
1530 """Prepares the target folder before we checkout.""" | 1220 """Prepares the target folder before we checkout.""" |
1531 dir_names = [sln.get('name') for sln in git_slns if 'name' in sln] | 1221 dir_names = [sln.get('name') for sln in git_slns if 'name' in sln] |
1532 # If we're active now, but the flag file doesn't exist (we weren't active | |
1533 # last run) or vice versa, blow away all checkouts. | |
1534 if bool(active) != bool(check_flag(options.flag_file)): | |
1535 ensure_no_checkout(dir_names, '*') | |
1536 if options.output_json: | 1222 if options.output_json: |
1537 # Make sure we tell recipes that we didn't run if the script exits here. | 1223 # Make sure we tell recipes that we didn't run if the script exits here. |
1538 emit_json(options.output_json, did_run=active) | 1224 emit_json(options.output_json, did_run=True) |
1539 if active: | 1225 if options.clobber: |
1540 if options.clobber: | 1226 ensure_no_checkout(dir_names, '*') |
1541 ensure_no_checkout(dir_names, '*') | |
1542 else: | |
1543 ensure_no_checkout(dir_names, '.svn') | |
1544 emit_flag(options.flag_file) | |
1545 else: | 1227 else: |
1546 delete_flag(options.flag_file) | 1228 ensure_no_checkout(dir_names, '.svn') |
1547 raise Inactive # This is caught in main() and we exit cleanly. | 1229 emit_flag(options.flag_file) |
1548 | 1230 |
1549 # Do a shallow checkout if the disk is less than 100GB. | 1231 # Do a shallow checkout if the disk is less than 100GB. |
1550 total_disk_space, free_disk_space = get_total_disk_space() | 1232 total_disk_space, free_disk_space = get_total_disk_space() |
1551 total_disk_space_gb = int(total_disk_space / (1024 * 1024 * 1024)) | 1233 total_disk_space_gb = int(total_disk_space / (1024 * 1024 * 1024)) |
1552 used_disk_space_gb = int((total_disk_space - free_disk_space) | 1234 used_disk_space_gb = int((total_disk_space - free_disk_space) |
1553 / (1024 * 1024 * 1024)) | 1235 / (1024 * 1024 * 1024)) |
1554 percent_used = int(used_disk_space_gb * 100 / total_disk_space_gb) | 1236 percent_used = int(used_disk_space_gb * 100 / total_disk_space_gb) |
1555 step_text = '[%dGB/%dGB used (%d%%)]' % (used_disk_space_gb, | 1237 step_text = '[%dGB/%dGB used (%d%%)]' % (used_disk_space_gb, |
1556 total_disk_space_gb, | 1238 total_disk_space_gb, |
1557 percent_used) | 1239 percent_used) |
1558 if not options.output_json: | 1240 if not options.output_json: |
1559 print '@@@STEP_TEXT@%s@@@' % step_text | 1241 print '@@@STEP_TEXT@%s@@@' % step_text |
1560 if not options.shallow: | 1242 if not options.shallow: |
1561 options.shallow = (total_disk_space < SHALLOW_CLONE_THRESHOLD | 1243 options.shallow = (total_disk_space < SHALLOW_CLONE_THRESHOLD |
1562 and not options.no_shallow) | 1244 and not options.no_shallow) |
1563 | 1245 |
1564 # The first solution is where the primary DEPS file resides. | 1246 # The first solution is where the primary DEPS file resides. |
1565 first_sln = dir_names[0] | 1247 first_sln = dir_names[0] |
1566 | 1248 |
1567 # Split all the revision specifications into a nice dict. | 1249 # Split all the revision specifications into a nice dict. |
1568 print 'Revisions: %s' % options.revision | 1250 print 'Revisions: %s' % options.revision |
1569 revisions = parse_revisions(options.revision, first_sln) | 1251 revisions = parse_revisions(options.revision, first_sln) |
1570 print 'Fetching Git checkout at %s@%s' % (first_sln, revisions[first_sln]) | 1252 print 'Fetching Git checkout at %s@%s' % (first_sln, revisions[first_sln]) |
1571 return revisions, step_text | 1253 return revisions, step_text |
1572 | 1254 |
1573 | 1255 |
1574 def checkout(options, git_slns, specs, buildspec, master, | 1256 def checkout(options, git_slns, specs, buildspec, |
1575 svn_root, revisions, step_text): | 1257 svn_root, revisions, step_text): |
1576 first_sln = git_slns[0]['name'] | 1258 first_sln = git_slns[0]['name'] |
1577 dir_names = [sln.get('name') for sln in git_slns if 'name' in sln] | 1259 dir_names = [sln.get('name') for sln in git_slns if 'name' in sln] |
1578 try: | 1260 try: |
1579 # Outer try is for catching patch failures and exiting gracefully. | 1261 # Outer try is for catching patch failures and exiting gracefully. |
1580 # Inner try is for catching gclient failures and retrying gracefully. | 1262 # Inner try is for catching gclient failures and retrying gracefully. |
1581 try: | 1263 try: |
1582 checkout_parameters = dict( | 1264 checkout_parameters = dict( |
1583 # First, pass in the base of what we want to check out. | 1265 # First, pass in the base of what we want to check out. |
1584 solutions=git_slns, | 1266 solutions=git_slns, |
(...skipping 41 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1626 patch_root=options.patch_root, | 1308 patch_root=options.patch_root, |
1627 patch_failure=True, | 1309 patch_failure=True, |
1628 step_text='%s PATCH FAILED' % step_text, | 1310 step_text='%s PATCH FAILED' % step_text, |
1629 fixed_revisions=revisions) | 1311 fixed_revisions=revisions) |
1630 else: | 1312 else: |
1631 # If we're not on recipes, tell annotator about our got_revisions. | 1313 # If we're not on recipes, tell annotator about our got_revisions. |
1632 emit_log_lines('patch error', e.output) | 1314 emit_log_lines('patch error', e.output) |
1633 print '@@@STEP_TEXT@%s PATCH FAILED@@@' % step_text | 1315 print '@@@STEP_TEXT@%s PATCH FAILED@@@' % step_text |
1634 raise | 1316 raise |
1635 | 1317 |
1636 # Revision is an svn revision, unless it's a git master. | |
1637 use_svn_rev = master not in GIT_MASTERS | |
1638 | |
1639 # Take care of got_revisions outputs. | 1318 # Take care of got_revisions outputs. |
1640 revision_mapping = dict(GOT_REVISION_MAPPINGS.get(svn_root, {})) | 1319 revision_mapping = dict(GOT_REVISION_MAPPINGS.get(svn_root, {})) |
1641 if options.revision_mapping: | 1320 if options.revision_mapping: |
1642 revision_mapping.update(options.revision_mapping) | 1321 revision_mapping.update(options.revision_mapping) |
1643 | 1322 |
1644 # If the repo is not in the default GOT_REVISION_MAPPINGS and no | 1323 # If the repo is not in the default GOT_REVISION_MAPPINGS and no |
1645 # revision_mapping were specified on the command line then | 1324 # revision_mapping were specified on the command line then |
1646 # default to setting 'got_revision' based on the first solution. | 1325 # default to setting 'got_revision' based on the first solution. |
1647 if not revision_mapping: | 1326 if not revision_mapping: |
1648 revision_mapping[first_sln] = 'got_revision' | 1327 revision_mapping[first_sln] = 'got_revision' |
1649 | 1328 |
1650 got_revisions = parse_got_revision(gclient_output, revision_mapping, | 1329 got_revisions = parse_got_revision(gclient_output, revision_mapping) |
1651 use_svn_rev) | |
1652 | 1330 |
1653 if not got_revisions: | 1331 if not got_revisions: |
1654 # TODO(hinoka): We should probably bail out here, but in the interest | 1332 # TODO(hinoka): We should probably bail out here, but in the interest |
1655 # of giving mis-configured bots some time to get fixed use a dummy | 1333 # of giving mis-configured bots some time to get fixed use a dummy |
1656 # revision here. | 1334 # revision here. |
1657 got_revisions = { 'got_revision': 'BOT_UPDATE_NO_REV_FOUND' } | 1335 got_revisions = { 'got_revision': 'BOT_UPDATE_NO_REV_FOUND' } |
1658 #raise Exception('No got_revision(s) found in gclient output') | 1336 #raise Exception('No got_revision(s) found in gclient output') |
1659 | 1337 |
1660 if options.output_json: | 1338 if options.output_json: |
1661 manifest = create_manifest() if options.output_manifest else None | 1339 manifest = create_manifest() if options.output_manifest else None |
1662 # Tell recipes information such as root, got_revision, etc. | 1340 # Tell recipes information such as root, got_revision, etc. |
1663 emit_json(options.output_json, | 1341 emit_json(options.output_json, |
1664 did_run=True, | 1342 did_run=True, |
1665 root=first_sln, | 1343 root=first_sln, |
1666 patch_root=options.patch_root, | 1344 patch_root=options.patch_root, |
1667 step_text=step_text, | 1345 step_text=step_text, |
1668 fixed_revisions=revisions, | 1346 fixed_revisions=revisions, |
1669 properties=got_revisions, | 1347 properties=got_revisions, |
1670 manifest=manifest) | 1348 manifest=manifest) |
1671 else: | 1349 else: |
1672 # If we're not on recipes, tell annotator about our got_revisions. | 1350 # If we're not on recipes, tell annotator about our got_revisions. |
1673 emit_properties(got_revisions) | 1351 emit_properties(got_revisions) |
1674 | 1352 |
1675 | 1353 |
1676 def print_help_text(force, output_json, active, master, builder, slave): | 1354 def print_help_text(master, builder, slave): |
1677 """Print helpful messages to tell devs whats going on.""" | 1355 """Print helpful messages to tell devs whats going on.""" |
1678 if force and output_json: | |
1679 recipe_force = 'Forced on by recipes' | |
1680 elif active and output_json: | |
1681 recipe_force = 'Off by recipes, but forced on by bot update' | |
1682 elif not active and output_json: | |
1683 recipe_force = 'Forced off by recipes' | |
1684 else: | |
1685 recipe_force = 'N/A. Was not called by recipes' | |
1686 | |
1687 print BOT_UPDATE_MESSAGE % { | 1356 print BOT_UPDATE_MESSAGE % { |
1688 'master': master or 'Not specified', | |
1689 'builder': builder or 'Not specified', | |
1690 'slave': slave or 'Not specified', | |
1691 'recipe': recipe_force, | |
1692 'CURRENT_DIR': CURRENT_DIR, | 1357 'CURRENT_DIR': CURRENT_DIR, |
1693 'BUILDER_DIR': BUILDER_DIR, | 1358 'BUILDER_DIR': BUILDER_DIR, |
1694 'SLAVE_DIR': SLAVE_DIR, | 1359 'SLAVE_DIR': SLAVE_DIR, |
1695 'THIS_DIR': THIS_DIR, | 1360 'THIS_DIR': THIS_DIR, |
1696 'SCRIPTS_DIR': SCRIPTS_DIR, | 1361 'SCRIPTS_DIR': SCRIPTS_DIR, |
1697 'BUILD_DIR': BUILD_DIR, | 1362 'BUILD_DIR': BUILD_DIR, |
1698 'ROOT_DIR': ROOT_DIR, | 1363 'ROOT_DIR': ROOT_DIR, |
1699 'DEPOT_TOOLS_DIR': DEPOT_TOOLS_DIR, | 1364 'DEPOT_TOOLS_DIR': DEPOT_TOOLS_DIR, |
1700 }, | 1365 }, |
1701 print ACTIVATED_MESSAGE if active else NOT_ACTIVATED_MESSAGE | |
1702 | 1366 |
1703 | 1367 |
1704 def main(): | 1368 def main(): |
1705 # Get inputs. | 1369 # Get inputs. |
1706 options, _ = parse_args() | 1370 options, _ = parse_args() |
1707 builder = options.builder_name | 1371 builder = options.builder_name |
1708 slave = options.slave_name | 1372 slave = options.slave_name |
1709 master = options.master | 1373 master = options.master |
1710 | 1374 |
1711 # Check if this script should activate or not. | 1375 # Prints some debugging information. |
1712 active = check_valid_host(master, builder, slave) or options.force or False | 1376 print_help_text(master, builder, slave) |
1713 | |
1714 # Print a helpful message to tell developers whats going on with this step. | |
1715 print_help_text( | |
1716 options.force, options.output_json, active, master, builder, slave) | |
1717 | 1377 |
1718 # Parse, munipulate, and print the gclient solutions. | 1378 # Parse, munipulate, and print the gclient solutions. |
1719 specs = {} | 1379 specs = {} |
1720 exec(options.specs, specs) | 1380 exec(options.specs, specs) |
1721 svn_solutions = specs.get('solutions', []) | 1381 svn_solutions = specs.get('solutions', []) |
1722 git_slns, svn_root, buildspec = solutions_to_git(svn_solutions) | 1382 git_slns, svn_root, buildspec = solutions_to_git(svn_solutions) |
1723 options.revision = maybe_ignore_revision(options.revision, buildspec) | 1383 options.revision = maybe_ignore_revision(options.revision, buildspec) |
1724 | 1384 |
1725 solutions_printer(git_slns) | 1385 solutions_printer(git_slns) |
1726 | 1386 |
1727 try: | 1387 try: |
1728 # Dun dun dun, the main part of bot_update. | 1388 # Dun dun dun, the main part of bot_update. |
1729 revisions, step_text = prepare(options, git_slns, active) | 1389 revisions, step_text = prepare(options, git_slns) |
1730 checkout(options, git_slns, specs, buildspec, master, svn_root, revisions, | 1390 checkout(options, git_slns, specs, buildspec, svn_root, revisions, |
1731 step_text) | 1391 step_text) |
1732 | 1392 |
1733 except Inactive: | |
1734 # Not active, should count as passing. | |
1735 pass | |
1736 except PatchFailed as e: | 1393 except PatchFailed as e: |
1737 emit_flag(options.flag_file) | 1394 emit_flag(options.flag_file) |
1738 # Return a specific non-zero exit code for patch failure (because it is | 1395 # Return a specific non-zero exit code for patch failure (because it is |
1739 # a failure), but make it different than other failures to distinguish | 1396 # a failure), but make it different than other failures to distinguish |
1740 # between infra failures (independent from patch author), and patch | 1397 # between infra failures (independent from patch author), and patch |
1741 # failures (that patch author can fix). However, PatchFailure due to | 1398 # failures (that patch author can fix). However, PatchFailure due to |
1742 # download patch failure is still an infra problem. | 1399 # download patch failure is still an infra problem. |
1743 if e.code == 3: | 1400 if e.code == 3: |
1744 # Patch download problem. | 1401 # Patch download problem. |
1745 return 87 | 1402 return 87 |
1746 # Genuine patch problem. | 1403 # Genuine patch problem. |
1747 return 88 | 1404 return 88 |
1748 except Exception: | 1405 except Exception: |
1749 # Unexpected failure. | 1406 # Unexpected failure. |
1750 emit_flag(options.flag_file) | 1407 emit_flag(options.flag_file) |
1751 raise | 1408 raise |
1752 else: | 1409 else: |
1753 emit_flag(options.flag_file) | 1410 emit_flag(options.flag_file) |
1754 | 1411 |
1755 | 1412 |
1756 if __name__ == '__main__': | 1413 if __name__ == '__main__': |
1757 sys.exit(main()) | 1414 sys.exit(main()) |
OLD | NEW |