Chromium Code Reviews| OLD | NEW |
|---|---|
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # Copyright 2014 The Chromium Authors. All rights reserved. | 2 # Copyright 2014 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
| 5 | 5 |
| 6 # TODO(hinoka): Use logging. | 6 # TODO(hinoka): Use logging. |
| 7 | 7 |
| 8 import cStringIO | 8 import cStringIO |
| 9 import codecs | 9 import codecs |
| 10 import collections | 10 import collections |
| (...skipping 11 matching lines...) Expand all Loading... | |
| 22 import time | 22 import time |
| 23 import urllib2 | 23 import urllib2 |
| 24 import urlparse | 24 import urlparse |
| 25 import uuid | 25 import uuid |
| 26 | 26 |
| 27 import os.path as path | 27 import os.path as path |
| 28 | 28 |
| 29 # How many bytes at a time to read from pipes. | 29 # How many bytes at a time to read from pipes. |
| 30 BUF_SIZE = 256 | 30 BUF_SIZE = 256 |
| 31 | 31 |
| 32 # Set this to true on flag day. | |
| 33 FLAG_DAY = False | |
| 34 | |
| 35 # Define a bunch of directory paths. | 32 # Define a bunch of directory paths. |
| 36 # Relative to the current working directory. | 33 # Relative to the current working directory. |
| 37 CURRENT_DIR = path.abspath(os.getcwd()) | 34 CURRENT_DIR = path.abspath(os.getcwd()) |
| 38 BUILDER_DIR = path.dirname(CURRENT_DIR) | 35 BUILDER_DIR = path.dirname(CURRENT_DIR) |
| 39 SLAVE_DIR = path.dirname(BUILDER_DIR) | 36 SLAVE_DIR = path.dirname(BUILDER_DIR) |
| 40 | 37 |
| 41 # Relative to this script's filesystem path. | 38 # Relative to this script's filesystem path. |
| 42 THIS_DIR = path.dirname(path.abspath(__file__)) | 39 THIS_DIR = path.dirname(path.abspath(__file__)) |
| 43 SCRIPTS_DIR = path.dirname(THIS_DIR) | 40 SCRIPTS_DIR = path.dirname(THIS_DIR) |
| 44 BUILD_DIR = path.dirname(SCRIPTS_DIR) | 41 BUILD_DIR = path.dirname(SCRIPTS_DIR) |
| (...skipping 231 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 276 DISABLED_BUILDERS = {} | 273 DISABLED_BUILDERS = {} |
| 277 DISABLED_BUILDERS.update(internal_data.get('DISABLED_BUILDERS', {})) | 274 DISABLED_BUILDERS.update(internal_data.get('DISABLED_BUILDERS', {})) |
| 278 | 275 |
| 279 DISABLED_SLAVES = {} | 276 DISABLED_SLAVES = {} |
| 280 DISABLED_SLAVES.update(internal_data.get('DISABLED_SLAVES', {})) | 277 DISABLED_SLAVES.update(internal_data.get('DISABLED_SLAVES', {})) |
| 281 | 278 |
| 282 HEAD_BUILDERS = {} | 279 HEAD_BUILDERS = {} |
| 283 HEAD_BUILDERS.update(internal_data.get('HEAD_BUILDERS', {})) | 280 HEAD_BUILDERS.update(internal_data.get('HEAD_BUILDERS', {})) |
| 284 | 281 |
| 285 # These masters work only in Git, meaning for got_revision, always output | 282 # These masters work only in Git, meaning for got_revision, always output |
| 286 # a git hash rather than a SVN rev. This goes away if flag_day is True. | 283 # a git hash rather than a SVN rev. |
| 287 GIT_MASTERS = [] | 284 GIT_MASTERS = [] |
| 288 GIT_MASTERS += internal_data.get('GIT_MASTERS', []) | 285 GIT_MASTERS += internal_data.get('GIT_MASTERS', []) |
|
zty
2014/10/10 18:11:28
Delete this ?
| |
| 289 | 286 |
| 290 | 287 |
| 291 # How many times to retry failed subprocess calls. | 288 # How many times to retry failed subprocess calls. |
| 292 RETRIES = 3 | 289 RETRIES = 3 |
| 293 | 290 |
| 294 # Find deps2git | 291 # Find deps2git |
| 295 DEPS2GIT_DIR_PATH = path.join(SCRIPTS_DIR, 'tools', 'deps2git') | 292 DEPS2GIT_DIR_PATH = path.join(SCRIPTS_DIR, 'tools', 'deps2git') |
| 296 DEPS2GIT_PATH = path.join(DEPS2GIT_DIR_PATH, 'deps2git.py') | 293 DEPS2GIT_PATH = path.join(DEPS2GIT_DIR_PATH, 'deps2git.py') |
| 297 S2G_INTERNAL_PATH = path.join(SCRIPTS_DIR, 'tools', 'deps2git_internal', | 294 S2G_INTERNAL_PATH = path.join(SCRIPTS_DIR, 'tools', 'deps2git_internal', |
| 298 'svn_to_git_internal.py') | 295 'svn_to_git_internal.py') |
| (...skipping 243 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 542 solution['deps_file'] = path.join(buildspec.container, buildspec.version, | 539 solution['deps_file'] = path.join(buildspec.container, buildspec.version, |
| 543 '.DEPS.git') | 540 '.DEPS.git') |
| 544 elif parsed_path in RECOGNIZED_PATHS: | 541 elif parsed_path in RECOGNIZED_PATHS: |
| 545 solution['url'] = RECOGNIZED_PATHS[parsed_path] | 542 solution['url'] = RECOGNIZED_PATHS[parsed_path] |
| 546 elif parsed_url.scheme == 'https' and 'googlesource' in parsed_url.netloc: | 543 elif parsed_url.scheme == 'https' and 'googlesource' in parsed_url.netloc: |
| 547 pass | 544 pass |
| 548 else: | 545 else: |
| 549 print 'Warning: %s' % ('path %r not recognized' % parsed_path,) | 546 print 'Warning: %s' % ('path %r not recognized' % parsed_path,) |
| 550 | 547 |
| 551 # Point .DEPS.git is the git version of the DEPS file. | 548 # Point .DEPS.git is the git version of the DEPS file. |
| 552 if not FLAG_DAY and not buildspec: | 549 if not buildspec: |
|
zty
2014/10/10 18:11:28
this if block should be deleted if FLAG_DAY is Tru
szager1
2014/10/10 18:18:10
No, it's still needed. We still sometimes do buil
| |
| 553 solution['deps_file'] = '.DEPS.git' | 550 solution['deps_file'] = '.DEPS.git' |
| 554 | 551 |
| 555 # Strip out deps containing $$V8_REV$$, etc. | 552 # Strip out deps containing $$V8_REV$$, etc. |
| 556 if 'custom_deps' in solution: | 553 if 'custom_deps' in solution: |
| 557 new_custom_deps = {} | 554 new_custom_deps = {} |
| 558 for deps_name, deps_value in solution['custom_deps'].iteritems(): | 555 for deps_name, deps_value in solution['custom_deps'].iteritems(): |
| 559 if deps_value and '$$' in deps_value: | 556 if deps_value and '$$' in deps_value: |
| 560 print 'Dropping %s:%s from custom deps' % (deps_name, deps_value) | 557 print 'Dropping %s:%s from custom deps' % (deps_name, deps_value) |
| 561 else: | 558 else: |
| 562 new_custom_deps[deps_name] = deps_value | 559 new_custom_deps[deps_name] = deps_value |
| (...skipping 145 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 708 cmd = ['log', '--format=%H', '--max-count=1', '--', filename] | 705 cmd = ['log', '--format=%H', '--max-count=1', '--', filename] |
| 709 return git(*cmd, cwd=repo_base).strip() | 706 return git(*cmd, cwd=repo_base).strip() |
| 710 | 707 |
| 711 | 708 |
| 712 def need_to_run_deps2git(repo_base, deps_file, deps_git_file): | 709 def need_to_run_deps2git(repo_base, deps_file, deps_git_file): |
| 713 """Checks to see if we need to run deps2git. | 710 """Checks to see if we need to run deps2git. |
| 714 | 711 |
| 715 Returns True if there was a DEPS change after the last .DEPS.git update | 712 Returns True if there was a DEPS change after the last .DEPS.git update |
| 716 or if DEPS has local modifications. | 713 or if DEPS has local modifications. |
| 717 """ | 714 """ |
| 718 print 'Checking if %s exists' % deps_git_file | |
| 719 if not path.isfile(deps_git_file): | |
| 720 # .DEPS.git doesn't exist but DEPS does? Generate one (unless we're past | |
| 721 # migration day, in which case we don't need .DEPS.git. | |
| 722 print 'it doesn\'t exist!' | |
| 723 return not FLAG_DAY | |
| 724 | |
| 725 # See if DEPS is dirty | 715 # See if DEPS is dirty |
| 726 deps_file_status = git( | 716 deps_file_status = git( |
| 727 'status', '--porcelain', deps_file, cwd=repo_base).strip() | 717 'status', '--porcelain', deps_file, cwd=repo_base).strip() |
| 728 if deps_file_status and deps_file_status.startswith('M '): | 718 if deps_file_status and deps_file_status.startswith('M '): |
| 729 return True | 719 return True |
| 730 | 720 |
| 731 last_known_deps_ref = _last_commit_for_file(deps_file, repo_base) | 721 last_known_deps_ref = _last_commit_for_file(deps_file, repo_base) |
| 732 last_known_deps_git_ref = _last_commit_for_file(deps_git_file, repo_base) | 722 last_known_deps_git_ref = _last_commit_for_file(deps_git_file, repo_base) |
| 733 merge_base_ref = git('merge-base', last_known_deps_ref, | 723 merge_base_ref = git('merge-base', last_known_deps_ref, |
| 734 last_known_deps_git_ref, cwd=repo_base).strip() | 724 last_known_deps_git_ref, cwd=repo_base).strip() |
| (...skipping 77 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 812 | 802 |
| 813 git_buildspec = get_git_buildspec(buildspec_path, buildspec_version) | 803 git_buildspec = get_git_buildspec(buildspec_path, buildspec_version) |
| 814 with open(deps_git_file, 'wb') as f: | 804 with open(deps_git_file, 'wb') as f: |
| 815 f.write(git_buildspec) | 805 f.write(git_buildspec) |
| 816 | 806 |
| 817 | 807 |
| 818 def ensure_deps2git(solution, shallow): | 808 def ensure_deps2git(solution, shallow): |
| 819 repo_base = path.join(os.getcwd(), solution['name']) | 809 repo_base = path.join(os.getcwd(), solution['name']) |
| 820 deps_file = path.join(repo_base, 'DEPS') | 810 deps_file = path.join(repo_base, 'DEPS') |
| 821 deps_git_file = path.join(repo_base, '.DEPS.git') | 811 deps_git_file = path.join(repo_base, '.DEPS.git') |
| 822 if not git('ls-files', 'DEPS', cwd=repo_base).strip(): | 812 if (not git('ls-files', 'DEPS', cwd=repo_base).strip() or |
| 813 not git('ls-files', '.DEPS.git', cwd=repo_base).strip()): | |
| 823 return | 814 return |
| 824 | 815 |
| 825 print 'Checking if %s is newer than %s' % (deps_file, deps_git_file) | 816 print 'Checking if %s is newer than %s' % (deps_file, deps_git_file) |
| 826 if not need_to_run_deps2git(repo_base, deps_file, deps_git_file): | 817 if not need_to_run_deps2git(repo_base, deps_file, deps_git_file): |
| 827 return | 818 return |
| 828 | 819 |
| 829 print '===DEPS file modified, need to run deps2git===' | 820 print '===DEPS file modified, need to run deps2git===' |
| 830 cmd = [sys.executable, DEPS2GIT_PATH, | 821 cmd = [sys.executable, DEPS2GIT_PATH, |
| 831 '--workspace', os.getcwd(), | 822 '--workspace', os.getcwd(), |
| 832 '--cache_dir', CACHE_DIR, | 823 '--cache_dir', CACHE_DIR, |
| (...skipping 581 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 1414 parse.add_option('--shallow', action='store_true', | 1405 parse.add_option('--shallow', action='store_true', |
| 1415 help='Use shallow clones for cache repositories.') | 1406 help='Use shallow clones for cache repositories.') |
| 1416 parse.add_option('--gyp_env', action='append', default=[], | 1407 parse.add_option('--gyp_env', action='append', default=[], |
| 1417 help='Environment variables to pass into gclient runhooks.') | 1408 help='Environment variables to pass into gclient runhooks.') |
| 1418 parse.add_option('--clobber', action='store_true', | 1409 parse.add_option('--clobber', action='store_true', |
| 1419 help='Delete checkout first, always') | 1410 help='Delete checkout first, always') |
| 1420 parse.add_option('--bot_update_clobber', action='store_true', dest='clobber', | 1411 parse.add_option('--bot_update_clobber', action='store_true', dest='clobber', |
| 1421 help='(synonym for --clobber)') | 1412 help='(synonym for --clobber)') |
| 1422 parse.add_option('-o', '--output_json', | 1413 parse.add_option('-o', '--output_json', |
| 1423 help='Output JSON information into a specified file') | 1414 help='Output JSON information into a specified file') |
| 1424 parse.add_option('--post-flag-day', action='store_true', | |
| 1425 help='Behave as if the chromium git migration has already ' | |
| 1426 'happened') | |
| 1427 parse.add_option('--no_shallow', action='store_true', | 1415 parse.add_option('--no_shallow', action='store_true', |
| 1428 help='Bypass disk detection and never shallow clone. ' | 1416 help='Bypass disk detection and never shallow clone. ' |
| 1429 'Does not override the --shallow flag') | 1417 'Does not override the --shallow flag') |
| 1430 parse.add_option('--no_runhooks', action='store_true', | 1418 parse.add_option('--no_runhooks', action='store_true', |
| 1431 help='Do not run hooks on official builder.') | 1419 help='Do not run hooks on official builder.') |
| 1432 parse.add_option('--refs', action='append', | 1420 parse.add_option('--refs', action='append', |
| 1433 help='Also fetch this refspec for the main solution(s). ' | 1421 help='Also fetch this refspec for the main solution(s). ' |
| 1434 'Eg. +refs/branch-heads/*') | 1422 'Eg. +refs/branch-heads/*') |
| 1435 parse.add_option('--with_branch_heads', action='store_true', | 1423 parse.add_option('--with_branch_heads', action='store_true', |
| 1436 help='Always pass --with_branch_heads to gclient. This ' | 1424 help='Always pass --with_branch_heads to gclient. This ' |
| 1437 'does the same thing as --refs +refs/branch-heads/*') | 1425 'does the same thing as --refs +refs/branch-heads/*') |
| 1438 | 1426 |
| 1439 | 1427 |
| 1440 options, args = parse.parse_args() | 1428 options, args = parse.parse_args() |
| 1441 | 1429 |
| 1442 if options.post_flag_day: | |
| 1443 global FLAG_DAY | |
| 1444 FLAG_DAY = True | |
| 1445 | |
| 1446 if not options.refs: | 1430 if not options.refs: |
| 1447 options.refs = [] | 1431 options.refs = [] |
| 1448 | 1432 |
| 1449 if options.with_branch_heads: | 1433 if options.with_branch_heads: |
| 1450 options.refs.append(BRANCH_HEADS_REFSPEC) | 1434 options.refs.append(BRANCH_HEADS_REFSPEC) |
| 1451 del options.with_branch_heads | 1435 del options.with_branch_heads |
| 1452 | 1436 |
| 1453 try: | 1437 try: |
| 1454 if options.revision_mapping_file: | 1438 if options.revision_mapping_file: |
| 1455 if options.revision_mapping: | 1439 if options.revision_mapping: |
| (...skipping 105 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 1561 log_lines=[('patch error', e.output),], | 1545 log_lines=[('patch error', e.output),], |
| 1562 patch_root=options.patch_root, | 1546 patch_root=options.patch_root, |
| 1563 patch_failure=True, | 1547 patch_failure=True, |
| 1564 step_text='%s PATCH FAILED' % step_text) | 1548 step_text='%s PATCH FAILED' % step_text) |
| 1565 else: | 1549 else: |
| 1566 # If we're not on recipes, tell annotator about our got_revisions. | 1550 # If we're not on recipes, tell annotator about our got_revisions. |
| 1567 emit_log_lines('patch error', e.output) | 1551 emit_log_lines('patch error', e.output) |
| 1568 print '@@@STEP_TEXT@%s PATCH FAILED@@@' % step_text | 1552 print '@@@STEP_TEXT@%s PATCH FAILED@@@' % step_text |
| 1569 raise | 1553 raise |
| 1570 | 1554 |
| 1571 # Revision is an svn revision, unless its a git master or past flag day. | 1555 # Revision is an svn revision, unless it's a git master. |
| 1572 use_svn_rev = master not in GIT_MASTERS and not FLAG_DAY | 1556 use_svn_rev = master not in GIT_MASTERS |
|
zty
2014/10/10 18:11:28
This should be deleted if FLAG_DAY is True. use_sv
szager1
2014/10/10 18:18:10
Hmm, I have to think about that. There may still
| |
| 1573 # Take care of got_revisions outputs. | 1557 # Take care of got_revisions outputs. |
| 1574 revision_mapping = dict(GOT_REVISION_MAPPINGS.get(svn_root, {})) | 1558 revision_mapping = dict(GOT_REVISION_MAPPINGS.get(svn_root, {})) |
| 1575 if options.revision_mapping: | 1559 if options.revision_mapping: |
| 1576 revision_mapping.update(options.revision_mapping) | 1560 revision_mapping.update(options.revision_mapping) |
| 1577 | 1561 |
| 1578 got_revisions = parse_got_revision(gclient_output, revision_mapping, | 1562 got_revisions = parse_got_revision(gclient_output, revision_mapping, |
| 1579 use_svn_rev) | 1563 use_svn_rev) |
| 1580 | 1564 |
| 1581 if not got_revisions and options.revision: | 1565 if not got_revisions and options.revision: |
| 1582 # If we have no revision_mapping, then just pass through the first revision | 1566 # If we have no revision_mapping, then just pass through the first revision |
| (...skipping 77 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 1660 except Exception: | 1644 except Exception: |
| 1661 # Unexpected failure. | 1645 # Unexpected failure. |
| 1662 emit_flag(options.flag_file) | 1646 emit_flag(options.flag_file) |
| 1663 raise | 1647 raise |
| 1664 else: | 1648 else: |
| 1665 emit_flag(options.flag_file) | 1649 emit_flag(options.flag_file) |
| 1666 | 1650 |
| 1667 | 1651 |
| 1668 if __name__ == '__main__': | 1652 if __name__ == '__main__': |
| 1669 sys.exit(main()) | 1653 sys.exit(main()) |
| OLD | NEW |