Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(132)

Side by Side Diff: scripts/slave/bot_update.py

Issue 642253002: Don't run deps2git if the repo doesn't have a .DEPS.git. (Closed) Base URL: svn://svn.chromium.org/chrome/trunk/tools/build
Patch Set: Created 6 years, 2 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « no previous file | tests/bot_update_test.py » ('j') | tests/bot_update_test.py » ('J')
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # Copyright 2014 The Chromium Authors. All rights reserved. 2 # Copyright 2014 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 5
6 # TODO(hinoka): Use logging. 6 # TODO(hinoka): Use logging.
7 7
8 import cStringIO 8 import cStringIO
9 import codecs 9 import codecs
10 import collections 10 import collections
(...skipping 11 matching lines...) Expand all
22 import time 22 import time
23 import urllib2 23 import urllib2
24 import urlparse 24 import urlparse
25 import uuid 25 import uuid
26 26
27 import os.path as path 27 import os.path as path
28 28
29 # How many bytes at a time to read from pipes. 29 # How many bytes at a time to read from pipes.
30 BUF_SIZE = 256 30 BUF_SIZE = 256
31 31
32 # Set this to true on flag day.
33 FLAG_DAY = False
34
35 # Define a bunch of directory paths. 32 # Define a bunch of directory paths.
36 # Relative to the current working directory. 33 # Relative to the current working directory.
37 CURRENT_DIR = path.abspath(os.getcwd()) 34 CURRENT_DIR = path.abspath(os.getcwd())
38 BUILDER_DIR = path.dirname(CURRENT_DIR) 35 BUILDER_DIR = path.dirname(CURRENT_DIR)
39 SLAVE_DIR = path.dirname(BUILDER_DIR) 36 SLAVE_DIR = path.dirname(BUILDER_DIR)
40 37
41 # Relative to this script's filesystem path. 38 # Relative to this script's filesystem path.
42 THIS_DIR = path.dirname(path.abspath(__file__)) 39 THIS_DIR = path.dirname(path.abspath(__file__))
43 SCRIPTS_DIR = path.dirname(THIS_DIR) 40 SCRIPTS_DIR = path.dirname(THIS_DIR)
44 BUILD_DIR = path.dirname(SCRIPTS_DIR) 41 BUILD_DIR = path.dirname(SCRIPTS_DIR)
(...skipping 231 matching lines...) Expand 10 before | Expand all | Expand 10 after
276 DISABLED_BUILDERS = {} 273 DISABLED_BUILDERS = {}
277 DISABLED_BUILDERS.update(internal_data.get('DISABLED_BUILDERS', {})) 274 DISABLED_BUILDERS.update(internal_data.get('DISABLED_BUILDERS', {}))
278 275
279 DISABLED_SLAVES = {} 276 DISABLED_SLAVES = {}
280 DISABLED_SLAVES.update(internal_data.get('DISABLED_SLAVES', {})) 277 DISABLED_SLAVES.update(internal_data.get('DISABLED_SLAVES', {}))
281 278
282 HEAD_BUILDERS = {} 279 HEAD_BUILDERS = {}
283 HEAD_BUILDERS.update(internal_data.get('HEAD_BUILDERS', {})) 280 HEAD_BUILDERS.update(internal_data.get('HEAD_BUILDERS', {}))
284 281
285 # These masters work only in Git, meaning for got_revision, always output 282 # These masters work only in Git, meaning for got_revision, always output
286 # a git hash rather than a SVN rev. This goes away if flag_day is True. 283 # a git hash rather than a SVN rev.
287 GIT_MASTERS = [] 284 GIT_MASTERS = []
288 GIT_MASTERS += internal_data.get('GIT_MASTERS', []) 285 GIT_MASTERS += internal_data.get('GIT_MASTERS', [])
zty 2014/10/10 18:11:28 Delete this ?
289 286
290 287
291 # How many times to retry failed subprocess calls. 288 # How many times to retry failed subprocess calls.
292 RETRIES = 3 289 RETRIES = 3
293 290
294 # Find deps2git 291 # Find deps2git
295 DEPS2GIT_DIR_PATH = path.join(SCRIPTS_DIR, 'tools', 'deps2git') 292 DEPS2GIT_DIR_PATH = path.join(SCRIPTS_DIR, 'tools', 'deps2git')
296 DEPS2GIT_PATH = path.join(DEPS2GIT_DIR_PATH, 'deps2git.py') 293 DEPS2GIT_PATH = path.join(DEPS2GIT_DIR_PATH, 'deps2git.py')
297 S2G_INTERNAL_PATH = path.join(SCRIPTS_DIR, 'tools', 'deps2git_internal', 294 S2G_INTERNAL_PATH = path.join(SCRIPTS_DIR, 'tools', 'deps2git_internal',
298 'svn_to_git_internal.py') 295 'svn_to_git_internal.py')
(...skipping 243 matching lines...) Expand 10 before | Expand all | Expand 10 after
542 solution['deps_file'] = path.join(buildspec.container, buildspec.version, 539 solution['deps_file'] = path.join(buildspec.container, buildspec.version,
543 '.DEPS.git') 540 '.DEPS.git')
544 elif parsed_path in RECOGNIZED_PATHS: 541 elif parsed_path in RECOGNIZED_PATHS:
545 solution['url'] = RECOGNIZED_PATHS[parsed_path] 542 solution['url'] = RECOGNIZED_PATHS[parsed_path]
546 elif parsed_url.scheme == 'https' and 'googlesource' in parsed_url.netloc: 543 elif parsed_url.scheme == 'https' and 'googlesource' in parsed_url.netloc:
547 pass 544 pass
548 else: 545 else:
549 print 'Warning: %s' % ('path %r not recognized' % parsed_path,) 546 print 'Warning: %s' % ('path %r not recognized' % parsed_path,)
550 547
551 # Point .DEPS.git is the git version of the DEPS file. 548 # Point .DEPS.git is the git version of the DEPS file.
552 if not FLAG_DAY and not buildspec: 549 if not buildspec:
zty 2014/10/10 18:11:28 this if block should be deleted if FLAG_DAY is Tru
szager1 2014/10/10 18:18:10 No, it's still needed. We still sometimes do buil
553 solution['deps_file'] = '.DEPS.git' 550 solution['deps_file'] = '.DEPS.git'
554 551
555 # Strip out deps containing $$V8_REV$$, etc. 552 # Strip out deps containing $$V8_REV$$, etc.
556 if 'custom_deps' in solution: 553 if 'custom_deps' in solution:
557 new_custom_deps = {} 554 new_custom_deps = {}
558 for deps_name, deps_value in solution['custom_deps'].iteritems(): 555 for deps_name, deps_value in solution['custom_deps'].iteritems():
559 if deps_value and '$$' in deps_value: 556 if deps_value and '$$' in deps_value:
560 print 'Dropping %s:%s from custom deps' % (deps_name, deps_value) 557 print 'Dropping %s:%s from custom deps' % (deps_name, deps_value)
561 else: 558 else:
562 new_custom_deps[deps_name] = deps_value 559 new_custom_deps[deps_name] = deps_value
(...skipping 145 matching lines...) Expand 10 before | Expand all | Expand 10 after
708 cmd = ['log', '--format=%H', '--max-count=1', '--', filename] 705 cmd = ['log', '--format=%H', '--max-count=1', '--', filename]
709 return git(*cmd, cwd=repo_base).strip() 706 return git(*cmd, cwd=repo_base).strip()
710 707
711 708
712 def need_to_run_deps2git(repo_base, deps_file, deps_git_file): 709 def need_to_run_deps2git(repo_base, deps_file, deps_git_file):
713 """Checks to see if we need to run deps2git. 710 """Checks to see if we need to run deps2git.
714 711
715 Returns True if there was a DEPS change after the last .DEPS.git update 712 Returns True if there was a DEPS change after the last .DEPS.git update
716 or if DEPS has local modifications. 713 or if DEPS has local modifications.
717 """ 714 """
718 print 'Checking if %s exists' % deps_git_file
719 if not path.isfile(deps_git_file):
720 # .DEPS.git doesn't exist but DEPS does? Generate one (unless we're past
721 # migration day, in which case we don't need .DEPS.git.
722 print 'it doesn\'t exist!'
723 return not FLAG_DAY
724
725 # See if DEPS is dirty 715 # See if DEPS is dirty
726 deps_file_status = git( 716 deps_file_status = git(
727 'status', '--porcelain', deps_file, cwd=repo_base).strip() 717 'status', '--porcelain', deps_file, cwd=repo_base).strip()
728 if deps_file_status and deps_file_status.startswith('M '): 718 if deps_file_status and deps_file_status.startswith('M '):
729 return True 719 return True
730 720
731 last_known_deps_ref = _last_commit_for_file(deps_file, repo_base) 721 last_known_deps_ref = _last_commit_for_file(deps_file, repo_base)
732 last_known_deps_git_ref = _last_commit_for_file(deps_git_file, repo_base) 722 last_known_deps_git_ref = _last_commit_for_file(deps_git_file, repo_base)
733 merge_base_ref = git('merge-base', last_known_deps_ref, 723 merge_base_ref = git('merge-base', last_known_deps_ref,
734 last_known_deps_git_ref, cwd=repo_base).strip() 724 last_known_deps_git_ref, cwd=repo_base).strip()
(...skipping 77 matching lines...) Expand 10 before | Expand all | Expand 10 after
812 802
813 git_buildspec = get_git_buildspec(buildspec_path, buildspec_version) 803 git_buildspec = get_git_buildspec(buildspec_path, buildspec_version)
814 with open(deps_git_file, 'wb') as f: 804 with open(deps_git_file, 'wb') as f:
815 f.write(git_buildspec) 805 f.write(git_buildspec)
816 806
817 807
818 def ensure_deps2git(solution, shallow): 808 def ensure_deps2git(solution, shallow):
819 repo_base = path.join(os.getcwd(), solution['name']) 809 repo_base = path.join(os.getcwd(), solution['name'])
820 deps_file = path.join(repo_base, 'DEPS') 810 deps_file = path.join(repo_base, 'DEPS')
821 deps_git_file = path.join(repo_base, '.DEPS.git') 811 deps_git_file = path.join(repo_base, '.DEPS.git')
822 if not git('ls-files', 'DEPS', cwd=repo_base).strip(): 812 if (not git('ls-files', 'DEPS', cwd=repo_base).strip() or
813 not git('ls-files', '.DEPS.git', cwd=repo_base).strip()):
823 return 814 return
824 815
825 print 'Checking if %s is newer than %s' % (deps_file, deps_git_file) 816 print 'Checking if %s is newer than %s' % (deps_file, deps_git_file)
826 if not need_to_run_deps2git(repo_base, deps_file, deps_git_file): 817 if not need_to_run_deps2git(repo_base, deps_file, deps_git_file):
827 return 818 return
828 819
829 print '===DEPS file modified, need to run deps2git===' 820 print '===DEPS file modified, need to run deps2git==='
830 cmd = [sys.executable, DEPS2GIT_PATH, 821 cmd = [sys.executable, DEPS2GIT_PATH,
831 '--workspace', os.getcwd(), 822 '--workspace', os.getcwd(),
832 '--cache_dir', CACHE_DIR, 823 '--cache_dir', CACHE_DIR,
(...skipping 581 matching lines...) Expand 10 before | Expand all | Expand 10 after
1414 parse.add_option('--shallow', action='store_true', 1405 parse.add_option('--shallow', action='store_true',
1415 help='Use shallow clones for cache repositories.') 1406 help='Use shallow clones for cache repositories.')
1416 parse.add_option('--gyp_env', action='append', default=[], 1407 parse.add_option('--gyp_env', action='append', default=[],
1417 help='Environment variables to pass into gclient runhooks.') 1408 help='Environment variables to pass into gclient runhooks.')
1418 parse.add_option('--clobber', action='store_true', 1409 parse.add_option('--clobber', action='store_true',
1419 help='Delete checkout first, always') 1410 help='Delete checkout first, always')
1420 parse.add_option('--bot_update_clobber', action='store_true', dest='clobber', 1411 parse.add_option('--bot_update_clobber', action='store_true', dest='clobber',
1421 help='(synonym for --clobber)') 1412 help='(synonym for --clobber)')
1422 parse.add_option('-o', '--output_json', 1413 parse.add_option('-o', '--output_json',
1423 help='Output JSON information into a specified file') 1414 help='Output JSON information into a specified file')
1424 parse.add_option('--post-flag-day', action='store_true',
1425 help='Behave as if the chromium git migration has already '
1426 'happened')
1427 parse.add_option('--no_shallow', action='store_true', 1415 parse.add_option('--no_shallow', action='store_true',
1428 help='Bypass disk detection and never shallow clone. ' 1416 help='Bypass disk detection and never shallow clone. '
1429 'Does not override the --shallow flag') 1417 'Does not override the --shallow flag')
1430 parse.add_option('--no_runhooks', action='store_true', 1418 parse.add_option('--no_runhooks', action='store_true',
1431 help='Do not run hooks on official builder.') 1419 help='Do not run hooks on official builder.')
1432 parse.add_option('--refs', action='append', 1420 parse.add_option('--refs', action='append',
1433 help='Also fetch this refspec for the main solution(s). ' 1421 help='Also fetch this refspec for the main solution(s). '
1434 'Eg. +refs/branch-heads/*') 1422 'Eg. +refs/branch-heads/*')
1435 parse.add_option('--with_branch_heads', action='store_true', 1423 parse.add_option('--with_branch_heads', action='store_true',
1436 help='Always pass --with_branch_heads to gclient. This ' 1424 help='Always pass --with_branch_heads to gclient. This '
1437 'does the same thing as --refs +refs/branch-heads/*') 1425 'does the same thing as --refs +refs/branch-heads/*')
1438 1426
1439 1427
1440 options, args = parse.parse_args() 1428 options, args = parse.parse_args()
1441 1429
1442 if options.post_flag_day:
1443 global FLAG_DAY
1444 FLAG_DAY = True
1445
1446 if not options.refs: 1430 if not options.refs:
1447 options.refs = [] 1431 options.refs = []
1448 1432
1449 if options.with_branch_heads: 1433 if options.with_branch_heads:
1450 options.refs.append(BRANCH_HEADS_REFSPEC) 1434 options.refs.append(BRANCH_HEADS_REFSPEC)
1451 del options.with_branch_heads 1435 del options.with_branch_heads
1452 1436
1453 try: 1437 try:
1454 if options.revision_mapping_file: 1438 if options.revision_mapping_file:
1455 if options.revision_mapping: 1439 if options.revision_mapping:
(...skipping 105 matching lines...) Expand 10 before | Expand all | Expand 10 after
1561 log_lines=[('patch error', e.output),], 1545 log_lines=[('patch error', e.output),],
1562 patch_root=options.patch_root, 1546 patch_root=options.patch_root,
1563 patch_failure=True, 1547 patch_failure=True,
1564 step_text='%s PATCH FAILED' % step_text) 1548 step_text='%s PATCH FAILED' % step_text)
1565 else: 1549 else:
1566 # If we're not on recipes, tell annotator about our got_revisions. 1550 # If we're not on recipes, tell annotator about our got_revisions.
1567 emit_log_lines('patch error', e.output) 1551 emit_log_lines('patch error', e.output)
1568 print '@@@STEP_TEXT@%s PATCH FAILED@@@' % step_text 1552 print '@@@STEP_TEXT@%s PATCH FAILED@@@' % step_text
1569 raise 1553 raise
1570 1554
1571 # Revision is an svn revision, unless its a git master or past flag day. 1555 # Revision is an svn revision, unless it's a git master.
1572 use_svn_rev = master not in GIT_MASTERS and not FLAG_DAY 1556 use_svn_rev = master not in GIT_MASTERS
zty 2014/10/10 18:11:28 This should be deleted if FLAG_DAY is True. use_sv
szager1 2014/10/10 18:18:10 Hmm, I have to think about that. There may still
1573 # Take care of got_revisions outputs. 1557 # Take care of got_revisions outputs.
1574 revision_mapping = dict(GOT_REVISION_MAPPINGS.get(svn_root, {})) 1558 revision_mapping = dict(GOT_REVISION_MAPPINGS.get(svn_root, {}))
1575 if options.revision_mapping: 1559 if options.revision_mapping:
1576 revision_mapping.update(options.revision_mapping) 1560 revision_mapping.update(options.revision_mapping)
1577 1561
1578 got_revisions = parse_got_revision(gclient_output, revision_mapping, 1562 got_revisions = parse_got_revision(gclient_output, revision_mapping,
1579 use_svn_rev) 1563 use_svn_rev)
1580 1564
1581 if not got_revisions and options.revision: 1565 if not got_revisions and options.revision:
1582 # If we have no revision_mapping, then just pass through the first revision 1566 # If we have no revision_mapping, then just pass through the first revision
(...skipping 77 matching lines...) Expand 10 before | Expand all | Expand 10 after
1660 except Exception: 1644 except Exception:
1661 # Unexpected failure. 1645 # Unexpected failure.
1662 emit_flag(options.flag_file) 1646 emit_flag(options.flag_file)
1663 raise 1647 raise
1664 else: 1648 else:
1665 emit_flag(options.flag_file) 1649 emit_flag(options.flag_file)
1666 1650
1667 1651
1668 if __name__ == '__main__': 1652 if __name__ == '__main__':
1669 sys.exit(main()) 1653 sys.exit(main())
OLDNEW
« no previous file with comments | « no previous file | tests/bot_update_test.py » ('j') | tests/bot_update_test.py » ('J')

Powered by Google App Engine
This is Rietveld 408576698