Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(242)

Side by Side Diff: recipe_modules/bot_update/resources/bot_update_lite.py

Issue 1954833002: Add bot_update_lite into recipes (Closed) Base URL: https://chromium.googlesource.com/chromium/tools/depot_tools.git@master
Patch Set: Remove expired args Created 4 years, 7 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # Copyright 2014 The Chromium Authors. All rights reserved. 2 # Copyright 2016 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 5
6 # TODO(hinoka): Use logging.
7
8 import cStringIO 6 import cStringIO
9 import codecs 7 import codecs
10 import collections 8 import collections
11 import copy 9 import copy
12 import ctypes 10 import ctypes
13 import json 11 import json
14 import optparse 12 import optparse
15 import os 13 import os
16 import pprint 14 import pprint
17 import random 15 import random
(...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after
66 '..', # ROOT_DIR 64 '..', # ROOT_DIR
67 'build', 65 'build',
68 'scripts'), 66 'scripts'),
69 path.join(SLAVE_DIR, '..', 'build', 'scripts'), 67 path.join(SLAVE_DIR, '..', 'build', 'scripts'),
70 ], default=path.dirname(THIS_DIR)) 68 ], default=path.dirname(THIS_DIR))
71 BUILD_DIR = path.dirname(SCRIPTS_DIR) 69 BUILD_DIR = path.dirname(SCRIPTS_DIR)
72 ROOT_DIR = path.dirname(BUILD_DIR) 70 ROOT_DIR = path.dirname(BUILD_DIR)
73 71
74 DEPOT_TOOLS_DIR = path.abspath(path.join(THIS_DIR, '..', '..', '..')) 72 DEPOT_TOOLS_DIR = path.abspath(path.join(THIS_DIR, '..', '..', '..'))
75 73
76 BUILD_INTERNAL_DIR = check_dir(
77 'build_internal', [
78 path.join(ROOT_DIR, 'build_internal'),
79 path.join(ROOT_DIR, # .recipe_deps
80 path.pardir, # slave
81 path.pardir, # scripts
82 path.pardir), # build_internal
83 ])
84
85
86 CHROMIUM_GIT_HOST = 'https://chromium.googlesource.com' 74 CHROMIUM_GIT_HOST = 'https://chromium.googlesource.com'
87 CHROMIUM_SRC_URL = CHROMIUM_GIT_HOST + '/chromium/src.git' 75 CHROMIUM_SRC_URL = CHROMIUM_GIT_HOST + '/chromium/src.git'
88 76
89 # Official builds use buildspecs, so this is a special case. 77 # Official builds use buildspecs, so this is a special case.
90 BUILDSPEC_TYPE = collections.namedtuple('buildspec', 78 BUILDSPEC_TYPE = collections.namedtuple('buildspec',
91 ('container', 'version')) 79 ('container', 'version'))
92 BUILDSPEC_RE = (r'^/chrome-internal/trunk/tools/buildspec/' 80 BUILDSPEC_RE = (r'^/chrome-internal/trunk/tools/buildspec/'
93 '(build|branches|releases)/(.+)$') 81 '(build|branches|releases)/(.+)$')
94 GIT_BUILDSPEC_PATH = ('https://chrome-internal.googlesource.com/chrome/tools/' 82 GIT_BUILDSPEC_PATH = ('https://chrome-internal.googlesource.com/chrome/tools/'
95 'buildspec') 83 'buildspec')
(...skipping 65 matching lines...) Expand 10 before | Expand all | Expand 10 after
161 'src/tools/swarm_client/': 'got_swarm_client_revision', 149 'src/tools/swarm_client/': 'got_swarm_client_revision',
162 'src/tools/swarming_client/': 'got_swarming_client_revision', 150 'src/tools/swarming_client/': 'got_swarming_client_revision',
163 'src/third_party/WebKit/': 'got_webkit_revision', 151 'src/third_party/WebKit/': 'got_webkit_revision',
164 'src/third_party/webrtc/': 'got_webrtc_revision', 152 'src/third_party/webrtc/': 'got_webrtc_revision',
165 'src/v8/': 'got_v8_revision', 153 'src/v8/': 'got_v8_revision',
166 } 154 }
167 } 155 }
168 156
169 157
170 BOT_UPDATE_MESSAGE = """ 158 BOT_UPDATE_MESSAGE = """
171 What is the "Bot Update" step? 159 Bot Update Debugging Information
172 ==============================
173
174 This step ensures that the source checkout on the bot (e.g. Chromium's src/ and
175 its dependencies) is checked out in a consistent state. This means that all of
176 the necessary repositories are checked out, no extra repositories are checked
177 out, and no locally modified files are present.
178
179 These actions used to be taken care of by the "gclient revert" and "update"
180 steps. However, those steps are known to be buggy and occasionally flaky. This
181 step has two main advantages over them:
182 * it only operates in Git, so the logic can be clearer and cleaner; and
183 * it is a slave-side script, so its behavior can be modified without
184 restarting the master.
185
186 Why Git, you ask? Because that is the direction that the Chromium project is
187 heading. This step is an integral part of the transition from using the SVN repo
188 at chrome/trunk/src to using the Git repo src.git. Please pardon the dust while
189 we fully convert everything to Git. This message will get out of your way
190 eventually, and the waterfall will be a happier place because of it.
191
192 This step can be activated or deactivated independently on every builder on
193 every master. When it is active, the "gclient revert" and "update" steps become
194 no-ops. When it is inactive, it prints this message, cleans up after itself, and
195 lets everything else continue as though nothing has changed. Eventually, when
196 everything is stable enough, this step will replace them entirely.
197 160
198 Debugging information: 161 Debugging information:
199 (master/builder/slave may be unspecified on recipes)
200 master: %(master)s
201 builder: %(builder)s
202 slave: %(slave)s
203 forced by recipes: %(recipe)s
204 CURRENT_DIR: %(CURRENT_DIR)s 162 CURRENT_DIR: %(CURRENT_DIR)s
205 BUILDER_DIR: %(BUILDER_DIR)s 163 BUILDER_DIR: %(BUILDER_DIR)s
206 SLAVE_DIR: %(SLAVE_DIR)s 164 SLAVE_DIR: %(SLAVE_DIR)s
207 THIS_DIR: %(THIS_DIR)s 165 THIS_DIR: %(THIS_DIR)s
208 SCRIPTS_DIR: %(SCRIPTS_DIR)s 166 SCRIPTS_DIR: %(SCRIPTS_DIR)s
209 BUILD_DIR: %(BUILD_DIR)s 167 BUILD_DIR: %(BUILD_DIR)s
210 ROOT_DIR: %(ROOT_DIR)s 168 ROOT_DIR: %(ROOT_DIR)s
211 DEPOT_TOOLS_DIR: %(DEPOT_TOOLS_DIR)s 169 DEPOT_TOOLS_DIR: %(DEPOT_TOOLS_DIR)s"""
212 bot_update.py is:"""
213
214 ACTIVATED_MESSAGE = """ACTIVE.
215 The bot will perform a Git checkout in this step.
216 The "gclient revert" and "update" steps are no-ops.
217
218 """
219
220 NOT_ACTIVATED_MESSAGE = """INACTIVE.
221 This step does nothing. You actually want to look at the "update" step.
222
223 """
224 170
225 171
226 GCLIENT_TEMPLATE = """solutions = %(solutions)s 172 GCLIENT_TEMPLATE = """solutions = %(solutions)s
227 173
228 cache_dir = r%(cache_dir)s 174 cache_dir = r%(cache_dir)s
229 %(target_os)s 175 %(target_os)s
230 %(target_os_only)s 176 %(target_os_only)s
231 """ 177 """
232 178
233 179
234 internal_data = {}
235 if BUILD_INTERNAL_DIR:
236 local_vars = {}
237 try:
238 execfile(os.path.join(
239 BUILD_INTERNAL_DIR, 'scripts', 'slave', 'bot_update_cfg.py'),
240 local_vars)
241 except Exception:
242 # Same as if BUILD_INTERNAL_DIR didn't exist in the first place.
243 print 'Warning: unable to read internal configuration file.'
244 print 'If this is an internal bot, this step may be erroneously inactive.'
245 internal_data = local_vars
246
247 RECOGNIZED_PATHS = {
248 # If SVN path matches key, the entire URL is rewritten to the Git url.
249 '/chrome/trunk/src':
250 CHROMIUM_SRC_URL,
251 '/chrome/trunk/src/tools/cros.DEPS':
252 CHROMIUM_GIT_HOST + '/chromium/src/tools/cros.DEPS.git',
253 }
254 RECOGNIZED_PATHS.update(internal_data.get('RECOGNIZED_PATHS', {}))
255
256 ENABLED_MASTERS = [
257 'bot_update.always_on',
258 'chromium.android',
259 'chromium.angle',
260 'chromium.chrome',
261 'chromium.chromedriver',
262 'chromium.chromiumos',
263 'chromium',
264 'chromium.fyi',
265 'chromium.goma',
266 'chromium.gpu',
267 'chromium.gpu.fyi',
268 'chromium.infra',
269 'chromium.infra.cron',
270 'chromium.linux',
271 'chromium.lkgr',
272 'chromium.mac',
273 'chromium.memory',
274 'chromium.memory.fyi',
275 'chromium.perf',
276 'chromium.perf.fyi',
277 'chromium.swarm',
278 'chromium.webkit',
279 'chromium.webrtc',
280 'chromium.webrtc.fyi',
281 'chromium.win',
282 'client.catapult',
283 'client.drmemory',
284 'client.mojo',
285 'client.nacl',
286 'client.nacl.ports',
287 'client.nacl.sdk',
288 'client.nacl.toolchain',
289 'client.pdfium',
290 'client.skia',
291 'client.skia.fyi',
292 'client.v8',
293 'client.v8.branches',
294 'client.v8.fyi',
295 'client.v8.ports',
296 'client.webrtc',
297 'client.webrtc.fyi',
298 'tryserver.blink',
299 'tryserver.client.catapult',
300 'tryserver.client.mojo',
301 'tryserver.chromium.android',
302 'tryserver.chromium.angle',
303 'tryserver.chromium.linux',
304 'tryserver.chromium.mac',
305 'tryserver.chromium.perf',
306 'tryserver.chromium.win',
307 'tryserver.infra',
308 'tryserver.nacl',
309 'tryserver.v8',
310 'tryserver.webrtc',
311 ]
312 ENABLED_MASTERS += internal_data.get('ENABLED_MASTERS', [])
313
314 ENABLED_BUILDERS = {
315 'client.dart.fyi': [
316 'v8-linux-release',
317 'v8-mac-release',
318 'v8-win-release',
319 ],
320 'client.dynamorio': [
321 'linux-v8-dr',
322 ],
323 }
324 ENABLED_BUILDERS.update(internal_data.get('ENABLED_BUILDERS', {}))
325
326 ENABLED_SLAVES = {}
327 ENABLED_SLAVES.update(internal_data.get('ENABLED_SLAVES', {}))
328
329 # Disabled filters get run AFTER enabled filters, so for example if a builder
330 # config is enabled, but a bot on that builder is disabled, that bot will
331 # be disabled.
332 DISABLED_BUILDERS = {}
333 DISABLED_BUILDERS.update(internal_data.get('DISABLED_BUILDERS', {}))
334
335 DISABLED_SLAVES = {}
336 DISABLED_SLAVES.update(internal_data.get('DISABLED_SLAVES', {}))
337
338 # These masters work only in Git, meaning for got_revision, always output
339 # a git hash rather than a SVN rev.
340 GIT_MASTERS = [
341 'client.v8',
342 'client.v8.branches',
343 'client.v8.ports',
344 'tryserver.v8',
345 ]
346 GIT_MASTERS += internal_data.get('GIT_MASTERS', [])
347
348
349 # How many times to try before giving up. 180 # How many times to try before giving up.
350 ATTEMPTS = 5 181 ATTEMPTS = 5
351 182
352 # Find deps2git
353 DEPS2GIT_DIR_PATH = path.join(SCRIPTS_DIR, 'tools', 'deps2git')
354 DEPS2GIT_PATH = path.join(DEPS2GIT_DIR_PATH, 'deps2git.py')
355 S2G_INTERNAL_PATH = path.join(SCRIPTS_DIR, 'tools', 'deps2git_internal',
356 'svn_to_git_internal.py')
357 GIT_CACHE_PATH = path.join(DEPOT_TOOLS_DIR, 'git_cache.py') 183 GIT_CACHE_PATH = path.join(DEPOT_TOOLS_DIR, 'git_cache.py')
358 184
359 # Find the patch tool. 185 # Find the patch tool.
360 if sys.platform.startswith('win'): 186 if sys.platform.startswith('win'):
361 if not BUILD_INTERNAL_DIR: 187 # TODO(hinoka): Check this in, is required for perf builders.
362 print 'Warning: could not find patch tool because there is no ' 188 PATCH_TOOL = path.join(THIS_DIR, 'patch.EXE')
363 print 'build_internal present.'
364 PATCH_TOOL = None
365 else:
366 PATCH_TOOL = path.join(BUILD_INTERNAL_DIR, 'tools', 'patch.EXE')
367 else: 189 else:
368 PATCH_TOOL = '/usr/bin/patch' 190 PATCH_TOOL = '/usr/bin/patch'
369 191
370 # If there is less than 100GB of disk space on the system, then we do 192 # If there is less than 100GB of disk space on the system, then we do
371 # a shallow checkout. 193 # a shallow checkout.
372 SHALLOW_CLONE_THRESHOLD = 100 * 1024 * 1024 * 1024 194 SHALLOW_CLONE_THRESHOLD = 100 * 1024 * 1024 * 1024
373 195
374 196
375 class SubprocessFailed(Exception): 197 class SubprocessFailed(Exception):
376 def __init__(self, message, code, output): 198 def __init__(self, message, code, output):
(...skipping 11 matching lines...) Expand all
388 210
389 211
390 class SVNRevisionNotFound(Exception): 212 class SVNRevisionNotFound(Exception):
391 pass 213 pass
392 214
393 215
394 class InvalidDiff(Exception): 216 class InvalidDiff(Exception):
395 pass 217 pass
396 218
397 219
398 class Inactive(Exception):
399 """Not really an exception, just used to exit early cleanly."""
400 pass
401
402
403 RETRY = object() 220 RETRY = object()
404 OK = object() 221 OK = object()
405 FAIL = object() 222 FAIL = object()
406 223
407 224
408 class PsPrinter(object): 225 class PsPrinter(object):
409 def __init__(self, interval=300): 226 def __init__(self, interval=300):
410 self.interval = interval 227 self.interval = interval
411 self.active = sys.platform.startswith('linux2') 228 self.active = sys.platform.startswith('linux2')
412 self.thread = None 229 self.thread = None
(...skipping 108 matching lines...) Expand 10 before | Expand all | Expand 10 after
521 338
522 def get_gclient_spec(solutions, target_os, target_os_only, git_cache_dir): 339 def get_gclient_spec(solutions, target_os, target_os_only, git_cache_dir):
523 return GCLIENT_TEMPLATE % { 340 return GCLIENT_TEMPLATE % {
524 'solutions': pprint.pformat(solutions, indent=4), 341 'solutions': pprint.pformat(solutions, indent=4),
525 'cache_dir': '"%s"' % git_cache_dir, 342 'cache_dir': '"%s"' % git_cache_dir,
526 'target_os': ('\ntarget_os=%s' % target_os) if target_os else '', 343 'target_os': ('\ntarget_os=%s' % target_os) if target_os else '',
527 'target_os_only': '\ntarget_os_only=%s' % target_os_only 344 'target_os_only': '\ntarget_os_only=%s' % target_os_only
528 } 345 }
529 346
530 347
531 def check_enabled(master, builder, slave):
532 if master in ENABLED_MASTERS:
533 return True
534 builder_list = ENABLED_BUILDERS.get(master)
535 if builder_list and builder in builder_list:
536 return True
537 slave_list = ENABLED_SLAVES.get(master)
538 if slave_list and slave in slave_list:
539 return True
540 return False
541
542
543 def check_disabled(master, builder, slave):
544 """Returns True if disabled, False if not disabled."""
545 builder_list = DISABLED_BUILDERS.get(master)
546 if builder_list and builder in builder_list:
547 return True
548 slave_list = DISABLED_SLAVES.get(master)
549 if slave_list and slave in slave_list:
550 return True
551 return False
552
553
554 def check_valid_host(master, builder, slave):
555 return (check_enabled(master, builder, slave)
556 and not check_disabled(master, builder, slave))
557
558
559 def maybe_ignore_revision(revision, buildspec): 348 def maybe_ignore_revision(revision, buildspec):
560 """Handle builders that don't care what buildbot tells them to build. 349 """Handle builders that don't care what buildbot tells them to build.
561 350
562 This is especially the case with branch builders that build from buildspecs 351 This is especially the case with branch builders that build from buildspecs
563 and/or trigger off multiple repositories, where the --revision passed in has 352 and/or trigger off multiple repositories, where the --revision passed in has
564 nothing to do with the solution being built. Clearing the revision in this 353 nothing to do with the solution being built. Clearing the revision in this
565 case causes bot_update to use HEAD rather that trying to checkout an 354 case causes bot_update to use HEAD rather that trying to checkout an
566 inappropriate version of the solution. 355 inappropriate version of the solution.
567 """ 356 """
568 if buildspec and buildspec.container == 'branches': 357 if buildspec and buildspec.container == 'branches':
(...skipping 233 matching lines...) Expand 10 before | Expand all | Expand 10 after
802 match = "^%s: [^ ]*@%s " % (GIT_SVN_ID_FOOTER_KEY, revision) 591 match = "^%s: [^ ]*@%s " % (GIT_SVN_ID_FOOTER_KEY, revision)
803 ref = branch if branch.startswith('refs/') else 'origin/%s' % branch 592 ref = branch if branch.startswith('refs/') else 'origin/%s' % branch
804 cmd = ['log', '-E', '--grep', match, '--format=%H', '--max-count=1', ref] 593 cmd = ['log', '-E', '--grep', match, '--format=%H', '--max-count=1', ref]
805 result = git(*cmd, cwd=sln_dir).strip() 594 result = git(*cmd, cwd=sln_dir).strip()
806 if result: 595 if result:
807 return result 596 return result
808 raise SVNRevisionNotFound('We can\'t resolve svn r%s into a git hash in %s' % 597 raise SVNRevisionNotFound('We can\'t resolve svn r%s into a git hash in %s' %
809 (revision, sln_dir)) 598 (revision, sln_dir))
810 599
811 600
812 def _last_commit_for_file(filename, repo_base):
813 cmd = ['log', '--format=%H', '--max-count=1', '--', filename]
814 return git(*cmd, cwd=repo_base).strip()
815
816
817 def need_to_run_deps2git(repo_base, deps_file, deps_git_file):
818 """Checks to see if we need to run deps2git.
819
820 Returns True if there was a DEPS change after the last .DEPS.git update
821 or if DEPS has local modifications.
822 """
823 # See if DEPS is dirty
824 deps_file_status = git(
825 'status', '--porcelain', deps_file, cwd=repo_base).strip()
826 if deps_file_status and deps_file_status.startswith('M '):
827 return True
828
829 last_known_deps_ref = _last_commit_for_file(deps_file, repo_base)
830 last_known_deps_git_ref = _last_commit_for_file(deps_git_file, repo_base)
831 merge_base_ref = git('merge-base', last_known_deps_ref,
832 last_known_deps_git_ref, cwd=repo_base).strip()
833
834 # If the merge base of the last DEPS and last .DEPS.git file is not
835 # equivilent to the hash of the last DEPS file, that means the DEPS file
836 # was committed after the last .DEPS.git file.
837 return last_known_deps_ref != merge_base_ref
838
839
840 def ensure_deps2git(solution, shallow, git_cache_dir):
841 repo_base = path.join(os.getcwd(), solution['name'])
842 deps_file = path.join(repo_base, 'DEPS')
843 deps_git_file = path.join(repo_base, '.DEPS.git')
844 if (not git('ls-files', 'DEPS', cwd=repo_base).strip() or
845 not git('ls-files', '.DEPS.git', cwd=repo_base).strip()):
846 return
847
848 print 'Checking if %s is newer than %s' % (deps_file, deps_git_file)
849 if not need_to_run_deps2git(repo_base, deps_file, deps_git_file):
850 return
851
852 print '===DEPS file modified, need to run deps2git==='
853 cmd = [sys.executable, DEPS2GIT_PATH,
854 '--workspace', os.getcwd(),
855 '--cache_dir', git_cache_dir,
856 '--deps', deps_file,
857 '--out', deps_git_file]
858 if 'chrome-internal.googlesource' in solution['url']:
859 cmd.extend(['--extra-rules', S2G_INTERNAL_PATH])
860 if shallow:
861 cmd.append('--shallow')
862 call(*cmd)
863
864
865 def emit_log_lines(name, lines): 601 def emit_log_lines(name, lines):
866 for line in lines.splitlines(): 602 for line in lines.splitlines():
867 print '@@@STEP_LOG_LINE@%s@%s@@@' % (name, line) 603 print '@@@STEP_LOG_LINE@%s@%s@@@' % (name, line)
868 print '@@@STEP_LOG_END@%s@@@' % name 604 print '@@@STEP_LOG_END@%s@@@' % name
869 605
870 606
871 def emit_properties(properties): 607 def emit_properties(properties):
872 for property_name, property_value in sorted(properties.items()): 608 for property_name, property_value in sorted(properties.items()):
873 print '@@@SET_BUILD_PROPERTY@%s@"%s"@@@' % (property_name, property_value) 609 print '@@@SET_BUILD_PROPERTY@%s@"%s"@@@' % (property_name, property_value)
874 610
(...skipping 125 matching lines...) Expand 10 before | Expand all | Expand 10 after
1000 736
1001 git('clean', '-dff', cwd=sln_dir) 737 git('clean', '-dff', cwd=sln_dir)
1002 738
1003 if first_solution: 739 if first_solution:
1004 git_ref = git('log', '--format=%H', '--max-count=1', 740 git_ref = git('log', '--format=%H', '--max-count=1',
1005 cwd=sln_dir).strip() 741 cwd=sln_dir).strip()
1006 first_solution = False 742 first_solution = False
1007 return git_ref 743 return git_ref
1008 744
1009 745
1010 def _download(url):
1011 """Fetch url and return content, with retries for flake."""
1012 for attempt in xrange(ATTEMPTS):
1013 try:
1014 return urllib2.urlopen(url).read()
1015 except Exception:
1016 if attempt == ATTEMPTS - 1:
1017 raise
1018
1019
1020 def parse_diff(diff): 746 def parse_diff(diff):
1021 """Takes a unified diff and returns a list of diffed files and their diffs. 747 """Takes a unified diff and returns a list of diffed files and their diffs.
1022 748
1023 The return format is a list of pairs of: 749 The return format is a list of pairs of:
1024 (<filename>, <diff contents>) 750 (<filename>, <diff contents>)
1025 <diff contents> is inclusive of the diff line. 751 <diff contents> is inclusive of the diff line.
1026 """ 752 """
1027 result = [] 753 result = []
1028 current_diff = '' 754 current_diff = ''
1029 current_header = None 755 current_header = None
(...skipping 290 matching lines...) Expand 10 before | Expand all | Expand 10 after
1320 target = '/'.join([relative_root, 'DEPS']).lstrip('/') 1046 target = '/'.join([relative_root, 'DEPS']).lstrip('/')
1321 if patches: 1047 if patches:
1322 apply_svn_patch(patch_root, patches, whitelist=[target]) 1048 apply_svn_patch(patch_root, patches, whitelist=[target])
1323 already_patched.append(target) 1049 already_patched.append(target)
1324 elif issue: 1050 elif issue:
1325 apply_rietveld_issue(issue, patchset, patch_root, rietveld_server, 1051 apply_rietveld_issue(issue, patchset, patch_root, rietveld_server,
1326 revision_mapping, git_ref, apply_issue_email_file, 1052 revision_mapping, git_ref, apply_issue_email_file,
1327 apply_issue_key_file, whitelist=[target]) 1053 apply_issue_key_file, whitelist=[target])
1328 already_patched.append(target) 1054 already_patched.append(target)
1329 1055
1330 if not buildspec:
1331 # Run deps2git if there is a DEPS change after the last .DEPS.git commit.
1332 for solution in solutions:
1333 ensure_deps2git(solution, shallow, git_cache_dir)
1334
1335 # Ensure our build/ directory is set up with the correct .gclient file. 1056 # Ensure our build/ directory is set up with the correct .gclient file.
1336 gclient_configure(solutions, target_os, target_os_only, git_cache_dir) 1057 gclient_configure(solutions, target_os, target_os_only, git_cache_dir)
1337 1058
1338 # Let gclient do the DEPS syncing. 1059 # Let gclient do the DEPS syncing.
1339 # The branch-head refspec is a special case because its possible Chrome 1060 # The branch-head refspec is a special case because its possible Chrome
1340 # src, which contains the branch-head refspecs, is DEPSed in. 1061 # src, which contains the branch-head refspecs, is DEPSed in.
1341 gclient_output = gclient_sync(buildspec or BRANCH_HEADS_REFSPEC in refs, 1062 gclient_output = gclient_sync(buildspec or BRANCH_HEADS_REFSPEC in refs,
1342 shallow) 1063 shallow)
1343 1064
1344 # Now that gclient_sync has finished, we should revert any .DEPS.git so that 1065 # Now that gclient_sync has finished, we should revert any .DEPS.git so that
(...skipping 83 matching lines...) Expand 10 before | Expand all | Expand 10 after
1428 1149
1429 parse.add_option('--issue', help='Issue number to patch from.') 1150 parse.add_option('--issue', help='Issue number to patch from.')
1430 parse.add_option('--patchset', 1151 parse.add_option('--patchset',
1431 help='Patchset from issue to patch from, if applicable.') 1152 help='Patchset from issue to patch from, if applicable.')
1432 parse.add_option('--apply_issue_email_file', 1153 parse.add_option('--apply_issue_email_file',
1433 help='--email-file option passthrough for apply_patch.py.') 1154 help='--email-file option passthrough for apply_patch.py.')
1434 parse.add_option('--apply_issue_key_file', 1155 parse.add_option('--apply_issue_key_file',
1435 help='--private-key-file option passthrough for ' 1156 help='--private-key-file option passthrough for '
1436 'apply_patch.py.') 1157 'apply_patch.py.')
1437 parse.add_option('--patch_url', help='Optional URL to SVN patch.') 1158 parse.add_option('--patch_url', help='Optional URL to SVN patch.')
1438 parse.add_option('--root', dest='patch_root',
1439 help='DEPRECATED: Use --patch_root.')
1440 parse.add_option('--patch_root', help='Directory to patch on top of.') 1159 parse.add_option('--patch_root', help='Directory to patch on top of.')
1441 parse.add_option('--rietveld_server', 1160 parse.add_option('--rietveld_server',
1442 default='codereview.chromium.org', 1161 default='codereview.chromium.org',
1443 help='Rietveld server.') 1162 help='Rietveld server.')
1444 parse.add_option('--gerrit_repo', 1163 parse.add_option('--gerrit_repo',
1445 help='Gerrit repository to pull the ref from.') 1164 help='Gerrit repository to pull the ref from.')
1446 parse.add_option('--gerrit_ref', help='Gerrit ref to apply.') 1165 parse.add_option('--gerrit_ref', help='Gerrit ref to apply.')
1447 parse.add_option('--gerrit_no_reset', action='store_true', 1166 parse.add_option('--gerrit_no_reset', action='store_true',
1448 help='Bypass calling reset after applying a gerrit ref.') 1167 help='Bypass calling reset after applying a gerrit ref.')
1449 parse.add_option('--specs', help='Gcilent spec.') 1168 parse.add_option('--specs', help='Gcilent spec.')
1450 parse.add_option('--master', help='Master name.')
1451 parse.add_option('-f', '--force', action='store_true',
1452 help='Bypass check to see if we want to be run. '
1453 'Should ONLY be used locally or by smart recipes.')
1454 parse.add_option('--revision_mapping', 1169 parse.add_option('--revision_mapping',
1455 help='{"path/to/repo/": "property_name"}') 1170 help='{"path/to/repo/": "property_name"}')
1456 parse.add_option('--revision_mapping_file', 1171 parse.add_option('--revision_mapping_file',
1457 help=('Same as revision_mapping, except its a path to a json' 1172 help=('Same as revision_mapping, except its a path to a json'
1458 ' file containing that format.')) 1173 ' file containing that format.'))
1459 parse.add_option('--revision', action='append', default=[], 1174 parse.add_option('--revision', action='append', default=[],
1460 help='Revision to check out. Can be an SVN revision number, ' 1175 help='Revision to check out. Can be an SVN revision number, '
1461 'git hash, or any form of git ref. Can prepend ' 1176 'git hash, or any form of git ref. Can prepend '
1462 'root@<rev> to specify which repository, where root ' 1177 'root@<rev> to specify which repository, where root '
1463 'is either a filesystem path, git https url, or ' 1178 'is either a filesystem path, git https url, or '
1464 'svn url. To specify Tip of Tree, set rev to HEAD.' 1179 'svn url. To specify Tip of Tree, set rev to HEAD.'
1465 'To specify a git branch and an SVN rev, <rev> can be ' 1180 'To specify a git branch and an SVN rev, <rev> can be '
1466 'set to <branch>:<revision>.') 1181 'set to <branch>:<revision>.')
1467 parse.add_option('--output_manifest', action='store_true', 1182 parse.add_option('--output_manifest', action='store_true',
1468 help=('Add manifest json to the json output.')) 1183 help=('Add manifest json to the json output.'))
1469 parse.add_option('--slave_name', default=socket.getfqdn().split('.')[0],
1470 help='Hostname of the current machine, '
1471 'used for determining whether or not to activate.')
1472 parse.add_option('--builder_name', help='Name of the builder, '
1473 'used for determining whether or not to activate.')
1474 parse.add_option('--build_dir', default=os.getcwd()) 1184 parse.add_option('--build_dir', default=os.getcwd())
1475 parse.add_option('--flag_file', default=path.join(os.getcwd(),
1476 'update.flag'))
1477 parse.add_option('--shallow', action='store_true', 1185 parse.add_option('--shallow', action='store_true',
1478 help='Use shallow clones for cache repositories.') 1186 help='Use shallow clones for cache repositories.')
1479 parse.add_option('--gyp_env', action='append', default=[], 1187 parse.add_option('--gyp_env', action='append', default=[],
1480 help='Environment variables to pass into gclient runhooks.') 1188 help='Environment variables to pass into gclient runhooks.')
1481 parse.add_option('--clobber', action='store_true', 1189 parse.add_option('--clobber', action='store_true',
1482 help='Delete checkout first, always') 1190 help='Delete checkout first, always')
1483 parse.add_option('--bot_update_clobber', action='store_true', dest='clobber', 1191 parse.add_option('--bot_update_clobber', action='store_true', dest='clobber',
1484 help='(synonym for --clobber)') 1192 help='(synonym for --clobber)')
1485 parse.add_option('-o', '--output_json', 1193 parse.add_option('-o', '--output_json',
1486 help='Output JSON information into a specified file') 1194 help='Output JSON information into a specified file')
(...skipping 38 matching lines...) Expand 10 before | Expand all | Expand 10 after
1525 1233
1526 # Because we print CACHE_DIR out into a .gclient file, and then later run 1234 # Because we print CACHE_DIR out into a .gclient file, and then later run
1527 # eval() on it, backslashes need to be escaped, otherwise "E:\b\build" gets 1235 # eval() on it, backslashes need to be escaped, otherwise "E:\b\build" gets
1528 # parsed as "E:[\x08][\x08]uild". 1236 # parsed as "E:[\x08][\x08]uild".
1529 if sys.platform.startswith('win'): 1237 if sys.platform.startswith('win'):
1530 options.git_cache_dir = options.git_cache_dir.replace('\\', '\\\\') 1238 options.git_cache_dir = options.git_cache_dir.replace('\\', '\\\\')
1531 1239
1532 return options, args 1240 return options, args
1533 1241
1534 1242
1535 def prepare(options, git_slns, active): 1243 def prepare(options, git_slns):
1536 """Prepares the target folder before we checkout.""" 1244 """Prepares the target folder before we checkout."""
1537 dir_names = [sln.get('name') for sln in git_slns if 'name' in sln] 1245 dir_names = [sln.get('name') for sln in git_slns if 'name' in sln]
1538 # If we're active now, but the flag file doesn't exist (we weren't active 1246 emit_json(options.output_json, did_run=True)
1539 # last run) or vice versa, blow away all checkouts. 1247 if options.clobber:
1540 if bool(active) != bool(check_flag(options.flag_file)):
1541 ensure_no_checkout(dir_names, '*') 1248 ensure_no_checkout(dir_names, '*')
1542 if options.output_json:
1543 # Make sure we tell recipes that we didn't run if the script exits here.
1544 emit_json(options.output_json, did_run=active)
1545 if active:
1546 if options.clobber:
1547 ensure_no_checkout(dir_names, '*')
1548 else:
1549 ensure_no_checkout(dir_names, '.svn')
1550 emit_flag(options.flag_file)
1551 else: 1249 else:
1552 delete_flag(options.flag_file) 1250 ensure_no_checkout(dir_names, '.svn')
1553 raise Inactive # This is caught in main() and we exit cleanly.
1554 1251
1555 # Do a shallow checkout if the disk is less than 100GB. 1252 # Do a shallow checkout if the disk is less than 100GB.
1556 total_disk_space, free_disk_space = get_total_disk_space() 1253 total_disk_space, free_disk_space = get_total_disk_space()
1557 total_disk_space_gb = int(total_disk_space / (1024 * 1024 * 1024)) 1254 total_disk_space_gb = int(total_disk_space / (1024 * 1024 * 1024))
1558 used_disk_space_gb = int((total_disk_space - free_disk_space) 1255 used_disk_space_gb = int((total_disk_space - free_disk_space)
1559 / (1024 * 1024 * 1024)) 1256 / (1024 * 1024 * 1024))
1560 percent_used = int(used_disk_space_gb * 100 / total_disk_space_gb) 1257 percent_used = int(used_disk_space_gb * 100 / total_disk_space_gb)
1561 step_text = '[%dGB/%dGB used (%d%%)]' % (used_disk_space_gb, 1258 step_text = '[%dGB/%dGB used (%d%%)]' % (used_disk_space_gb,
1562 total_disk_space_gb, 1259 total_disk_space_gb,
1563 percent_used) 1260 percent_used)
1564 if not options.output_json: 1261 if not options.output_json:
1565 print '@@@STEP_TEXT@%s@@@' % step_text 1262 print '@@@STEP_TEXT@%s@@@' % step_text
1566 if not options.shallow: 1263 if not options.shallow:
1567 options.shallow = (total_disk_space < SHALLOW_CLONE_THRESHOLD 1264 options.shallow = (total_disk_space < SHALLOW_CLONE_THRESHOLD
1568 and not options.no_shallow) 1265 and not options.no_shallow)
1569 1266
1570 # The first solution is where the primary DEPS file resides. 1267 # The first solution is where the primary DEPS file resides.
1571 first_sln = dir_names[0] 1268 first_sln = dir_names[0]
1572 1269
1573 # Split all the revision specifications into a nice dict. 1270 # Split all the revision specifications into a nice dict.
1574 print 'Revisions: %s' % options.revision 1271 print 'Revisions: %s' % options.revision
1575 revisions = parse_revisions(options.revision, first_sln) 1272 revisions = parse_revisions(options.revision, first_sln)
1576 print 'Fetching Git checkout at %s@%s' % (first_sln, revisions[first_sln]) 1273 print 'Fetching Git checkout at %s@%s' % (first_sln, revisions[first_sln])
1577 return revisions, step_text 1274 return revisions, step_text
1578 1275
1579 1276
1580 def checkout(options, git_slns, specs, buildspec, master, 1277 def checkout(options, git_slns, specs, buildspec,
1581 svn_root, revisions, step_text): 1278 svn_root, revisions, step_text):
1582 first_sln = git_slns[0]['name'] 1279 first_sln = git_slns[0]['name']
1583 dir_names = [sln.get('name') for sln in git_slns if 'name' in sln] 1280 dir_names = [sln.get('name') for sln in git_slns if 'name' in sln]
1584 try: 1281 try:
1585 # Outer try is for catching patch failures and exiting gracefully. 1282 # Outer try is for catching patch failures and exiting gracefully.
1586 # Inner try is for catching gclient failures and retrying gracefully. 1283 # Inner try is for catching gclient failures and retrying gracefully.
1587 try: 1284 try:
1588 checkout_parameters = dict( 1285 checkout_parameters = dict(
1589 # First, pass in the base of what we want to check out. 1286 # First, pass in the base of what we want to check out.
1590 solutions=git_slns, 1287 solutions=git_slns,
(...skipping 42 matching lines...) Expand 10 before | Expand all | Expand 10 after
1633 patch_root=options.patch_root, 1330 patch_root=options.patch_root,
1634 patch_failure=True, 1331 patch_failure=True,
1635 step_text='%s PATCH FAILED' % step_text, 1332 step_text='%s PATCH FAILED' % step_text,
1636 fixed_revisions=revisions) 1333 fixed_revisions=revisions)
1637 else: 1334 else:
1638 # If we're not on recipes, tell annotator about our got_revisions. 1335 # If we're not on recipes, tell annotator about our got_revisions.
1639 emit_log_lines('patch error', e.output) 1336 emit_log_lines('patch error', e.output)
1640 print '@@@STEP_TEXT@%s PATCH FAILED@@@' % step_text 1337 print '@@@STEP_TEXT@%s PATCH FAILED@@@' % step_text
1641 raise 1338 raise
1642 1339
1643 # Revision is an svn revision, unless it's a git master.
1644 use_svn_rev = master not in GIT_MASTERS
1645
1646 # Take care of got_revisions outputs. 1340 # Take care of got_revisions outputs.
1647 revision_mapping = dict(GOT_REVISION_MAPPINGS.get(svn_root, {})) 1341 revision_mapping = dict(GOT_REVISION_MAPPINGS.get(svn_root, {}))
1648 if options.revision_mapping: 1342 if options.revision_mapping:
1649 revision_mapping.update(options.revision_mapping) 1343 revision_mapping.update(options.revision_mapping)
1650 1344
1651 # If the repo is not in the default GOT_REVISION_MAPPINGS and no 1345 # If the repo is not in the default GOT_REVISION_MAPPINGS and no
1652 # revision_mapping were specified on the command line then 1346 # revision_mapping were specified on the command line then
1653 # default to setting 'got_revision' based on the first solution. 1347 # default to setting 'got_revision' based on the first solution.
1654 if not revision_mapping: 1348 if not revision_mapping:
1655 revision_mapping[first_sln] = 'got_revision' 1349 revision_mapping[first_sln] = 'got_revision'
1656 1350
1657 got_revisions = parse_got_revision(gclient_output, revision_mapping, 1351 got_revisions = parse_got_revision(gclient_output, revision_mapping)
1658 use_svn_rev)
1659 1352
1660 if not got_revisions: 1353 if not got_revisions:
1661 # TODO(hinoka): We should probably bail out here, but in the interest 1354 # TODO(hinoka): We should probably bail out here, but in the interest
1662 # of giving mis-configured bots some time to get fixed use a dummy 1355 # of giving mis-configured bots some time to get fixed use a dummy
1663 # revision here. 1356 # revision here.
1664 got_revisions = { 'got_revision': 'BOT_UPDATE_NO_REV_FOUND' } 1357 got_revisions = { 'got_revision': 'BOT_UPDATE_NO_REV_FOUND' }
1665 #raise Exception('No got_revision(s) found in gclient output') 1358 #raise Exception('No got_revision(s) found in gclient output')
1666 1359
1667 if options.output_json: 1360 if options.output_json:
1668 manifest = create_manifest() if options.output_manifest else None 1361 manifest = create_manifest() if options.output_manifest else None
1669 # Tell recipes information such as root, got_revision, etc. 1362 # Tell recipes information such as root, got_revision, etc.
1670 emit_json(options.output_json, 1363 emit_json(options.output_json,
1671 did_run=True, 1364 did_run=True,
1672 root=first_sln, 1365 root=first_sln,
1673 patch_root=options.patch_root, 1366 patch_root=options.patch_root,
1674 step_text=step_text, 1367 step_text=step_text,
1675 fixed_revisions=revisions, 1368 fixed_revisions=revisions,
1676 properties=got_revisions, 1369 properties=got_revisions,
1677 manifest=manifest) 1370 manifest=manifest)
1678 else: 1371 else:
1679 # If we're not on recipes, tell annotator about our got_revisions. 1372 # If we're not on recipes, tell annotator about our got_revisions.
1680 emit_properties(got_revisions) 1373 emit_properties(got_revisions)
1681 1374
1682 1375
1683 def print_help_text(force, output_json, active, master, builder, slave): 1376 def print_help_text():
1684 """Print helpful messages to tell devs whats going on.""" 1377 """Print helpful messages to tell devs whats going on."""
1685 if force and output_json:
1686 recipe_force = 'Forced on by recipes'
1687 elif active and output_json:
1688 recipe_force = 'Off by recipes, but forced on by bot update'
1689 elif not active and output_json:
1690 recipe_force = 'Forced off by recipes'
1691 else:
1692 recipe_force = 'N/A. Was not called by recipes'
1693
1694 print BOT_UPDATE_MESSAGE % { 1378 print BOT_UPDATE_MESSAGE % {
1695 'master': master or 'Not specified',
1696 'builder': builder or 'Not specified',
1697 'slave': slave or 'Not specified',
1698 'recipe': recipe_force,
1699 'CURRENT_DIR': CURRENT_DIR, 1379 'CURRENT_DIR': CURRENT_DIR,
1700 'BUILDER_DIR': BUILDER_DIR, 1380 'BUILDER_DIR': BUILDER_DIR,
1701 'SLAVE_DIR': SLAVE_DIR, 1381 'SLAVE_DIR': SLAVE_DIR,
1702 'THIS_DIR': THIS_DIR, 1382 'THIS_DIR': THIS_DIR,
1703 'SCRIPTS_DIR': SCRIPTS_DIR, 1383 'SCRIPTS_DIR': SCRIPTS_DIR,
1704 'BUILD_DIR': BUILD_DIR, 1384 'BUILD_DIR': BUILD_DIR,
1705 'ROOT_DIR': ROOT_DIR, 1385 'ROOT_DIR': ROOT_DIR,
1706 'DEPOT_TOOLS_DIR': DEPOT_TOOLS_DIR, 1386 'DEPOT_TOOLS_DIR': DEPOT_TOOLS_DIR,
1707 }, 1387 }
1708 print ACTIVATED_MESSAGE if active else NOT_ACTIVATED_MESSAGE
1709 1388
1710 1389
1711 def main(): 1390 def main():
1712 # Get inputs. 1391 # Get inputs.
1713 options, _ = parse_args() 1392 options, _ = parse_args()
1714 builder = options.builder_name
1715 slave = options.slave_name
1716 master = options.master
1717
1718 # Check if this script should activate or not.
1719 active = check_valid_host(master, builder, slave) or options.force or False
1720 1393
1721 # Print a helpful message to tell developers whats going on with this step. 1394 # Print a helpful message to tell developers whats going on with this step.
1722 print_help_text( 1395 print_help_text()
1723 options.force, options.output_json, active, master, builder, slave)
1724 1396
1725 # Parse, munipulate, and print the gclient solutions. 1397 # Parse, munipulate, and print the gclient solutions.
1726 specs = {} 1398 specs = {}
1727 exec(options.specs, specs) 1399 exec(options.specs, specs)
1728 svn_solutions = specs.get('solutions', []) 1400 svn_solutions = specs.get('solutions', [])
1729 git_slns, svn_root, buildspec = solutions_to_git(svn_solutions) 1401 git_slns, svn_root, buildspec = solutions_to_git(svn_solutions)
1730 options.revision = maybe_ignore_revision(options.revision, buildspec) 1402 options.revision = maybe_ignore_revision(options.revision, buildspec)
1731 1403
1732 solutions_printer(git_slns) 1404 solutions_printer(git_slns)
1733 1405
1734 try: 1406 try:
1735 # Dun dun dun, the main part of bot_update. 1407 # Dun dun dun, the main part of bot_update.
1736 revisions, step_text = prepare(options, git_slns, active) 1408 revisions, step_text = prepare(options, git_slns)
1737 checkout(options, git_slns, specs, buildspec, master, svn_root, revisions, 1409 checkout(options, git_slns, specs, buildspec, svn_root, revisions,
1738 step_text) 1410 step_text)
1739 1411
1740 except Inactive:
1741 # Not active, should count as passing.
1742 pass
1743 except PatchFailed as e: 1412 except PatchFailed as e:
1744 emit_flag(options.flag_file) 1413 emit_flag(options.flag_file)
1745 # Return a specific non-zero exit code for patch failure (because it is 1414 # Return a specific non-zero exit code for patch failure (because it is
1746 # a failure), but make it different than other failures to distinguish 1415 # a failure), but make it different than other failures to distinguish
1747 # between infra failures (independent from patch author), and patch 1416 # between infra failures (independent from patch author), and patch
1748 # failures (that patch author can fix). However, PatchFailure due to 1417 # failures (that patch author can fix). However, PatchFailure due to
1749 # download patch failure is still an infra problem. 1418 # download patch failure is still an infra problem.
1750 if e.code == 3: 1419 if e.code == 3:
1751 # Patch download problem. 1420 # Patch download problem.
1752 return 87 1421 return 87
1753 # Genuine patch problem. 1422 # Genuine patch problem.
1754 return 88 1423 return 88
1755 except Exception: 1424 except Exception:
1756 # Unexpected failure. 1425 # Unexpected failure.
1757 emit_flag(options.flag_file) 1426 emit_flag(options.flag_file)
1758 raise 1427 raise
1759 else: 1428 else:
1760 emit_flag(options.flag_file) 1429 emit_flag(options.flag_file)
1761 1430
1762 1431
1763 if __name__ == '__main__': 1432 if __name__ == '__main__':
1764 sys.exit(main()) 1433 sys.exit(main())
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698