OLD | NEW |
---|---|
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright 2014 The Chromium Authors. All rights reserved. | 2 # Copyright 2014 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 # TODO(hinoka): Use logging. | 6 # TODO(hinoka): Use logging. |
7 | 7 |
8 import cStringIO | 8 import cStringIO |
9 import codecs | 9 import codecs |
10 import collections | 10 import collections |
(...skipping 11 matching lines...) Expand all Loading... | |
22 import time | 22 import time |
23 import urllib2 | 23 import urllib2 |
24 import urlparse | 24 import urlparse |
25 import uuid | 25 import uuid |
26 | 26 |
27 import os.path as path | 27 import os.path as path |
28 | 28 |
29 # How many bytes at a time to read from pipes. | 29 # How many bytes at a time to read from pipes. |
30 BUF_SIZE = 256 | 30 BUF_SIZE = 256 |
31 | 31 |
32 # Set this to true on flag day. | |
33 FLAG_DAY = False | |
34 | |
35 # Define a bunch of directory paths. | 32 # Define a bunch of directory paths. |
36 # Relative to the current working directory. | 33 # Relative to the current working directory. |
37 CURRENT_DIR = path.abspath(os.getcwd()) | 34 CURRENT_DIR = path.abspath(os.getcwd()) |
38 BUILDER_DIR = path.dirname(CURRENT_DIR) | 35 BUILDER_DIR = path.dirname(CURRENT_DIR) |
39 SLAVE_DIR = path.dirname(BUILDER_DIR) | 36 SLAVE_DIR = path.dirname(BUILDER_DIR) |
40 | 37 |
41 # Relative to this script's filesystem path. | 38 # Relative to this script's filesystem path. |
42 THIS_DIR = path.dirname(path.abspath(__file__)) | 39 THIS_DIR = path.dirname(path.abspath(__file__)) |
43 SCRIPTS_DIR = path.dirname(THIS_DIR) | 40 SCRIPTS_DIR = path.dirname(THIS_DIR) |
44 BUILD_DIR = path.dirname(SCRIPTS_DIR) | 41 BUILD_DIR = path.dirname(SCRIPTS_DIR) |
(...skipping 231 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
276 DISABLED_BUILDERS = {} | 273 DISABLED_BUILDERS = {} |
277 DISABLED_BUILDERS.update(internal_data.get('DISABLED_BUILDERS', {})) | 274 DISABLED_BUILDERS.update(internal_data.get('DISABLED_BUILDERS', {})) |
278 | 275 |
279 DISABLED_SLAVES = {} | 276 DISABLED_SLAVES = {} |
280 DISABLED_SLAVES.update(internal_data.get('DISABLED_SLAVES', {})) | 277 DISABLED_SLAVES.update(internal_data.get('DISABLED_SLAVES', {})) |
281 | 278 |
282 HEAD_BUILDERS = {} | 279 HEAD_BUILDERS = {} |
283 HEAD_BUILDERS.update(internal_data.get('HEAD_BUILDERS', {})) | 280 HEAD_BUILDERS.update(internal_data.get('HEAD_BUILDERS', {})) |
284 | 281 |
285 # These masters work only in Git, meaning for got_revision, always output | 282 # These masters work only in Git, meaning for got_revision, always output |
286 # a git hash rather than a SVN rev. This goes away if flag_day is True. | 283 # a git hash rather than a SVN rev. |
287 GIT_MASTERS = [] | 284 GIT_MASTERS = [] |
288 GIT_MASTERS += internal_data.get('GIT_MASTERS', []) | 285 GIT_MASTERS += internal_data.get('GIT_MASTERS', []) |
zty
2014/10/10 18:11:28
Delete this ?
| |
289 | 286 |
290 | 287 |
291 # How many times to retry failed subprocess calls. | 288 # How many times to retry failed subprocess calls. |
292 RETRIES = 3 | 289 RETRIES = 3 |
293 | 290 |
294 # Find deps2git | 291 # Find deps2git |
295 DEPS2GIT_DIR_PATH = path.join(SCRIPTS_DIR, 'tools', 'deps2git') | 292 DEPS2GIT_DIR_PATH = path.join(SCRIPTS_DIR, 'tools', 'deps2git') |
296 DEPS2GIT_PATH = path.join(DEPS2GIT_DIR_PATH, 'deps2git.py') | 293 DEPS2GIT_PATH = path.join(DEPS2GIT_DIR_PATH, 'deps2git.py') |
297 S2G_INTERNAL_PATH = path.join(SCRIPTS_DIR, 'tools', 'deps2git_internal', | 294 S2G_INTERNAL_PATH = path.join(SCRIPTS_DIR, 'tools', 'deps2git_internal', |
298 'svn_to_git_internal.py') | 295 'svn_to_git_internal.py') |
(...skipping 243 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
542 solution['deps_file'] = path.join(buildspec.container, buildspec.version, | 539 solution['deps_file'] = path.join(buildspec.container, buildspec.version, |
543 '.DEPS.git') | 540 '.DEPS.git') |
544 elif parsed_path in RECOGNIZED_PATHS: | 541 elif parsed_path in RECOGNIZED_PATHS: |
545 solution['url'] = RECOGNIZED_PATHS[parsed_path] | 542 solution['url'] = RECOGNIZED_PATHS[parsed_path] |
546 elif parsed_url.scheme == 'https' and 'googlesource' in parsed_url.netloc: | 543 elif parsed_url.scheme == 'https' and 'googlesource' in parsed_url.netloc: |
547 pass | 544 pass |
548 else: | 545 else: |
549 print 'Warning: %s' % ('path %r not recognized' % parsed_path,) | 546 print 'Warning: %s' % ('path %r not recognized' % parsed_path,) |
550 | 547 |
551 # Point .DEPS.git is the git version of the DEPS file. | 548 # Point .DEPS.git is the git version of the DEPS file. |
552 if not FLAG_DAY and not buildspec: | 549 if not buildspec: |
zty
2014/10/10 18:11:28
this if block should be deleted if FLAG_DAY is Tru
szager1
2014/10/10 18:18:10
No, it's still needed. We still sometimes do buil
| |
553 solution['deps_file'] = '.DEPS.git' | 550 solution['deps_file'] = '.DEPS.git' |
554 | 551 |
555 # Strip out deps containing $$V8_REV$$, etc. | 552 # Strip out deps containing $$V8_REV$$, etc. |
556 if 'custom_deps' in solution: | 553 if 'custom_deps' in solution: |
557 new_custom_deps = {} | 554 new_custom_deps = {} |
558 for deps_name, deps_value in solution['custom_deps'].iteritems(): | 555 for deps_name, deps_value in solution['custom_deps'].iteritems(): |
559 if deps_value and '$$' in deps_value: | 556 if deps_value and '$$' in deps_value: |
560 print 'Dropping %s:%s from custom deps' % (deps_name, deps_value) | 557 print 'Dropping %s:%s from custom deps' % (deps_name, deps_value) |
561 else: | 558 else: |
562 new_custom_deps[deps_name] = deps_value | 559 new_custom_deps[deps_name] = deps_value |
(...skipping 145 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
708 cmd = ['log', '--format=%H', '--max-count=1', '--', filename] | 705 cmd = ['log', '--format=%H', '--max-count=1', '--', filename] |
709 return git(*cmd, cwd=repo_base).strip() | 706 return git(*cmd, cwd=repo_base).strip() |
710 | 707 |
711 | 708 |
712 def need_to_run_deps2git(repo_base, deps_file, deps_git_file): | 709 def need_to_run_deps2git(repo_base, deps_file, deps_git_file): |
713 """Checks to see if we need to run deps2git. | 710 """Checks to see if we need to run deps2git. |
714 | 711 |
715 Returns True if there was a DEPS change after the last .DEPS.git update | 712 Returns True if there was a DEPS change after the last .DEPS.git update |
716 or if DEPS has local modifications. | 713 or if DEPS has local modifications. |
717 """ | 714 """ |
718 print 'Checking if %s exists' % deps_git_file | |
719 if not path.isfile(deps_git_file): | |
720 # .DEPS.git doesn't exist but DEPS does? Generate one (unless we're past | |
721 # migration day, in which case we don't need .DEPS.git. | |
722 print 'it doesn\'t exist!' | |
723 return not FLAG_DAY | |
724 | |
725 # See if DEPS is dirty | 715 # See if DEPS is dirty |
726 deps_file_status = git( | 716 deps_file_status = git( |
727 'status', '--porcelain', deps_file, cwd=repo_base).strip() | 717 'status', '--porcelain', deps_file, cwd=repo_base).strip() |
728 if deps_file_status and deps_file_status.startswith('M '): | 718 if deps_file_status and deps_file_status.startswith('M '): |
729 return True | 719 return True |
730 | 720 |
731 last_known_deps_ref = _last_commit_for_file(deps_file, repo_base) | 721 last_known_deps_ref = _last_commit_for_file(deps_file, repo_base) |
732 last_known_deps_git_ref = _last_commit_for_file(deps_git_file, repo_base) | 722 last_known_deps_git_ref = _last_commit_for_file(deps_git_file, repo_base) |
733 merge_base_ref = git('merge-base', last_known_deps_ref, | 723 merge_base_ref = git('merge-base', last_known_deps_ref, |
734 last_known_deps_git_ref, cwd=repo_base).strip() | 724 last_known_deps_git_ref, cwd=repo_base).strip() |
(...skipping 77 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
812 | 802 |
813 git_buildspec = get_git_buildspec(buildspec_path, buildspec_version) | 803 git_buildspec = get_git_buildspec(buildspec_path, buildspec_version) |
814 with open(deps_git_file, 'wb') as f: | 804 with open(deps_git_file, 'wb') as f: |
815 f.write(git_buildspec) | 805 f.write(git_buildspec) |
816 | 806 |
817 | 807 |
818 def ensure_deps2git(solution, shallow): | 808 def ensure_deps2git(solution, shallow): |
819 repo_base = path.join(os.getcwd(), solution['name']) | 809 repo_base = path.join(os.getcwd(), solution['name']) |
820 deps_file = path.join(repo_base, 'DEPS') | 810 deps_file = path.join(repo_base, 'DEPS') |
821 deps_git_file = path.join(repo_base, '.DEPS.git') | 811 deps_git_file = path.join(repo_base, '.DEPS.git') |
822 if not git('ls-files', 'DEPS', cwd=repo_base).strip(): | 812 if (not git('ls-files', 'DEPS', cwd=repo_base).strip() or |
813 not git('ls-files', '.DEPS.git', cwd=repo_base).strip()): | |
823 return | 814 return |
824 | 815 |
825 print 'Checking if %s is newer than %s' % (deps_file, deps_git_file) | 816 print 'Checking if %s is newer than %s' % (deps_file, deps_git_file) |
826 if not need_to_run_deps2git(repo_base, deps_file, deps_git_file): | 817 if not need_to_run_deps2git(repo_base, deps_file, deps_git_file): |
827 return | 818 return |
828 | 819 |
829 print '===DEPS file modified, need to run deps2git===' | 820 print '===DEPS file modified, need to run deps2git===' |
830 cmd = [sys.executable, DEPS2GIT_PATH, | 821 cmd = [sys.executable, DEPS2GIT_PATH, |
831 '--workspace', os.getcwd(), | 822 '--workspace', os.getcwd(), |
832 '--cache_dir', CACHE_DIR, | 823 '--cache_dir', CACHE_DIR, |
(...skipping 581 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
1414 parse.add_option('--shallow', action='store_true', | 1405 parse.add_option('--shallow', action='store_true', |
1415 help='Use shallow clones for cache repositories.') | 1406 help='Use shallow clones for cache repositories.') |
1416 parse.add_option('--gyp_env', action='append', default=[], | 1407 parse.add_option('--gyp_env', action='append', default=[], |
1417 help='Environment variables to pass into gclient runhooks.') | 1408 help='Environment variables to pass into gclient runhooks.') |
1418 parse.add_option('--clobber', action='store_true', | 1409 parse.add_option('--clobber', action='store_true', |
1419 help='Delete checkout first, always') | 1410 help='Delete checkout first, always') |
1420 parse.add_option('--bot_update_clobber', action='store_true', dest='clobber', | 1411 parse.add_option('--bot_update_clobber', action='store_true', dest='clobber', |
1421 help='(synonym for --clobber)') | 1412 help='(synonym for --clobber)') |
1422 parse.add_option('-o', '--output_json', | 1413 parse.add_option('-o', '--output_json', |
1423 help='Output JSON information into a specified file') | 1414 help='Output JSON information into a specified file') |
1424 parse.add_option('--post-flag-day', action='store_true', | |
1425 help='Behave as if the chromium git migration has already ' | |
1426 'happened') | |
1427 parse.add_option('--no_shallow', action='store_true', | 1415 parse.add_option('--no_shallow', action='store_true', |
1428 help='Bypass disk detection and never shallow clone. ' | 1416 help='Bypass disk detection and never shallow clone. ' |
1429 'Does not override the --shallow flag') | 1417 'Does not override the --shallow flag') |
1430 parse.add_option('--no_runhooks', action='store_true', | 1418 parse.add_option('--no_runhooks', action='store_true', |
1431 help='Do not run hooks on official builder.') | 1419 help='Do not run hooks on official builder.') |
1432 parse.add_option('--refs', action='append', | 1420 parse.add_option('--refs', action='append', |
1433 help='Also fetch this refspec for the main solution(s). ' | 1421 help='Also fetch this refspec for the main solution(s). ' |
1434 'Eg. +refs/branch-heads/*') | 1422 'Eg. +refs/branch-heads/*') |
1435 parse.add_option('--with_branch_heads', action='store_true', | 1423 parse.add_option('--with_branch_heads', action='store_true', |
1436 help='Always pass --with_branch_heads to gclient. This ' | 1424 help='Always pass --with_branch_heads to gclient. This ' |
1437 'does the same thing as --refs +refs/branch-heads/*') | 1425 'does the same thing as --refs +refs/branch-heads/*') |
1438 | 1426 |
1439 | 1427 |
1440 options, args = parse.parse_args() | 1428 options, args = parse.parse_args() |
1441 | 1429 |
1442 if options.post_flag_day: | |
1443 global FLAG_DAY | |
1444 FLAG_DAY = True | |
1445 | |
1446 if not options.refs: | 1430 if not options.refs: |
1447 options.refs = [] | 1431 options.refs = [] |
1448 | 1432 |
1449 if options.with_branch_heads: | 1433 if options.with_branch_heads: |
1450 options.refs.append(BRANCH_HEADS_REFSPEC) | 1434 options.refs.append(BRANCH_HEADS_REFSPEC) |
1451 del options.with_branch_heads | 1435 del options.with_branch_heads |
1452 | 1436 |
1453 try: | 1437 try: |
1454 if options.revision_mapping_file: | 1438 if options.revision_mapping_file: |
1455 if options.revision_mapping: | 1439 if options.revision_mapping: |
(...skipping 105 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
1561 log_lines=[('patch error', e.output),], | 1545 log_lines=[('patch error', e.output),], |
1562 patch_root=options.patch_root, | 1546 patch_root=options.patch_root, |
1563 patch_failure=True, | 1547 patch_failure=True, |
1564 step_text='%s PATCH FAILED' % step_text) | 1548 step_text='%s PATCH FAILED' % step_text) |
1565 else: | 1549 else: |
1566 # If we're not on recipes, tell annotator about our got_revisions. | 1550 # If we're not on recipes, tell annotator about our got_revisions. |
1567 emit_log_lines('patch error', e.output) | 1551 emit_log_lines('patch error', e.output) |
1568 print '@@@STEP_TEXT@%s PATCH FAILED@@@' % step_text | 1552 print '@@@STEP_TEXT@%s PATCH FAILED@@@' % step_text |
1569 raise | 1553 raise |
1570 | 1554 |
1571 # Revision is an svn revision, unless its a git master or past flag day. | 1555 # Revision is an svn revision, unless it's a git master. |
1572 use_svn_rev = master not in GIT_MASTERS and not FLAG_DAY | 1556 use_svn_rev = master not in GIT_MASTERS |
zty
2014/10/10 18:11:28
This should be deleted if FLAG_DAY is True. use_sv
szager1
2014/10/10 18:18:10
Hmm, I have to think about that. There may still
| |
1573 # Take care of got_revisions outputs. | 1557 # Take care of got_revisions outputs. |
1574 revision_mapping = dict(GOT_REVISION_MAPPINGS.get(svn_root, {})) | 1558 revision_mapping = dict(GOT_REVISION_MAPPINGS.get(svn_root, {})) |
1575 if options.revision_mapping: | 1559 if options.revision_mapping: |
1576 revision_mapping.update(options.revision_mapping) | 1560 revision_mapping.update(options.revision_mapping) |
1577 | 1561 |
1578 got_revisions = parse_got_revision(gclient_output, revision_mapping, | 1562 got_revisions = parse_got_revision(gclient_output, revision_mapping, |
1579 use_svn_rev) | 1563 use_svn_rev) |
1580 | 1564 |
1581 if not got_revisions and options.revision: | 1565 if not got_revisions and options.revision: |
1582 # If we have no revision_mapping, then just pass through the first revision | 1566 # If we have no revision_mapping, then just pass through the first revision |
(...skipping 77 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
1660 except Exception: | 1644 except Exception: |
1661 # Unexpected failure. | 1645 # Unexpected failure. |
1662 emit_flag(options.flag_file) | 1646 emit_flag(options.flag_file) |
1663 raise | 1647 raise |
1664 else: | 1648 else: |
1665 emit_flag(options.flag_file) | 1649 emit_flag(options.flag_file) |
1666 | 1650 |
1667 | 1651 |
1668 if __name__ == '__main__': | 1652 if __name__ == '__main__': |
1669 sys.exit(main()) | 1653 sys.exit(main()) |
OLD | NEW |