| OLD | NEW |
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # Copyright 2014 The Chromium Authors. All rights reserved. | 2 # Copyright 2014 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
| 5 | 5 |
| 6 # TODO(hinoka): Use logging. | 6 # TODO(hinoka): Use logging. |
| 7 | 7 |
| 8 import cStringIO | 8 import cStringIO |
| 9 import codecs | 9 import codecs |
| 10 import collections | 10 import collections |
| (...skipping 14 matching lines...) Expand all Loading... |
| 25 import urllib2 | 25 import urllib2 |
| 26 import urlparse | 26 import urlparse |
| 27 import uuid | 27 import uuid |
| 28 | 28 |
| 29 import os.path as path | 29 import os.path as path |
| 30 | 30 |
| 31 # How many bytes at a time to read from pipes. | 31 # How many bytes at a time to read from pipes. |
| 32 BUF_SIZE = 256 | 32 BUF_SIZE = 256 |
| 33 | 33 |
| 34 | 34 |
| 35 # TODO(luqui): This is a horrible hack to identify build_internal when build | |
| 36 # is a recipe dependency. bot_update should not be depending on internal, | |
| 37 # rather the arrow should go the other way (or just be destroyed). | |
| 38 def check_dir(name, dirs, default=None): | |
| 39 for d in dirs: | |
| 40 d = path.abspath(d) | |
| 41 if path.basename(d) == name and path.isdir(d): | |
| 42 return d | |
| 43 return default | |
| 44 | |
| 45 | |
| 46 # Define a bunch of directory paths. | 35 # Define a bunch of directory paths. |
| 47 # Relative to the current working directory. | 36 # Relative to the current working directory. |
| 48 CURRENT_DIR = path.abspath(os.getcwd()) | 37 CURRENT_DIR = path.abspath(os.getcwd()) |
| 49 BUILDER_DIR = path.dirname(CURRENT_DIR) | 38 BUILDER_DIR = path.dirname(CURRENT_DIR) |
| 50 SLAVE_DIR = path.dirname(BUILDER_DIR) | |
| 51 | 39 |
| 52 # Relative to this script's filesystem path. | 40 # Relative to this script's filesystem path. |
| 53 THIS_DIR = path.dirname(path.abspath(__file__)) | 41 THIS_DIR = path.dirname(path.abspath(__file__)) |
| 54 SCRIPTS_DIR = check_dir( | |
| 55 'scripts', [ | |
| 56 path.dirname(THIS_DIR), | |
| 57 path.join(SLAVE_DIR, '..', 'scripts'), | |
| 58 path.join(THIS_DIR, # resources | |
| 59 '..', # bot_update | |
| 60 '..', # recipe_modules | |
| 61 '..', # depot_tools | |
| 62 '..', # .recipe_deps | |
| 63 '..', # slave | |
| 64 '..', # scripts | |
| 65 '..', # build_internal | |
| 66 '..', # ROOT_DIR | |
| 67 'build', | |
| 68 'scripts'), | |
| 69 path.join(SLAVE_DIR, '..', 'build', 'scripts'), | |
| 70 ], default=path.dirname(THIS_DIR)) | |
| 71 BUILD_DIR = path.dirname(SCRIPTS_DIR) | |
| 72 ROOT_DIR = path.dirname(BUILD_DIR) | |
| 73 | 42 |
| 74 DEPOT_TOOLS_DIR = path.abspath(path.join(THIS_DIR, '..', '..', '..')) | 43 DEPOT_TOOLS_DIR = path.abspath(path.join(THIS_DIR, '..', '..', '..')) |
| 75 | 44 |
| 76 BUILD_INTERNAL_DIR = check_dir( | |
| 77 'build_internal', [ | |
| 78 path.join(ROOT_DIR, 'build_internal'), | |
| 79 path.join(ROOT_DIR, # .recipe_deps | |
| 80 path.pardir, # slave | |
| 81 path.pardir, # scripts | |
| 82 path.pardir), # build_internal | |
| 83 ]) | |
| 84 | |
| 85 | |
| 86 CHROMIUM_GIT_HOST = 'https://chromium.googlesource.com' | 45 CHROMIUM_GIT_HOST = 'https://chromium.googlesource.com' |
| 87 CHROMIUM_SRC_URL = CHROMIUM_GIT_HOST + '/chromium/src.git' | 46 CHROMIUM_SRC_URL = CHROMIUM_GIT_HOST + '/chromium/src.git' |
| 88 | 47 |
| 89 BRANCH_HEADS_REFSPEC = '+refs/branch-heads/*' | 48 BRANCH_HEADS_REFSPEC = '+refs/branch-heads/*' |
| 90 | 49 |
| 91 # Regular expression that matches a single commit footer line. | 50 # Regular expression that matches a single commit footer line. |
| 92 COMMIT_FOOTER_ENTRY_RE = re.compile(r'([^:]+):\s+(.+)') | 51 COMMIT_FOOTER_ENTRY_RE = re.compile(r'([^:]+):\s+(.+)') |
| 93 | 52 |
| 94 # Footer metadata keys for regular and gsubtreed mirrored commit positions. | 53 # Footer metadata keys for regular and gsubtreed mirrored commit positions. |
| 95 COMMIT_POSITION_FOOTER_KEY = 'Cr-Commit-Position' | 54 COMMIT_POSITION_FOOTER_KEY = 'Cr-Commit-Position' |
| (...skipping 18 matching lines...) Expand all Loading... |
| 114 | 73 |
| 115 | 74 |
| 116 GCLIENT_TEMPLATE = """solutions = %(solutions)s | 75 GCLIENT_TEMPLATE = """solutions = %(solutions)s |
| 117 | 76 |
| 118 cache_dir = r%(cache_dir)s | 77 cache_dir = r%(cache_dir)s |
| 119 %(target_os)s | 78 %(target_os)s |
| 120 %(target_os_only)s | 79 %(target_os_only)s |
| 121 """ | 80 """ |
| 122 | 81 |
| 123 | 82 |
| 124 internal_data = {} | |
| 125 if BUILD_INTERNAL_DIR: | |
| 126 local_vars = {} | |
| 127 try: | |
| 128 execfile(os.path.join( | |
| 129 BUILD_INTERNAL_DIR, 'scripts', 'slave', 'bot_update_cfg.py'), | |
| 130 local_vars) | |
| 131 except Exception: | |
| 132 # Same as if BUILD_INTERNAL_DIR didn't exist in the first place. | |
| 133 print 'Warning: unable to read internal configuration file.' | |
| 134 print 'If this is an internal bot, this step may be erroneously inactive.' | |
| 135 internal_data = local_vars | |
| 136 | |
| 137 | |
| 138 | |
| 139 # How many times to try before giving up. | 83 # How many times to try before giving up. |
| 140 ATTEMPTS = 5 | 84 ATTEMPTS = 5 |
| 141 | 85 |
| 142 GIT_CACHE_PATH = path.join(DEPOT_TOOLS_DIR, 'git_cache.py') | 86 GIT_CACHE_PATH = path.join(DEPOT_TOOLS_DIR, 'git_cache.py') |
| 143 | 87 |
| 144 # Find the patch tool. | |
| 145 if sys.platform.startswith('win'): | |
| 146 if not BUILD_INTERNAL_DIR: | |
| 147 print 'Warning: could not find patch tool because there is no ' | |
| 148 print 'build_internal present.' | |
| 149 PATCH_TOOL = None | |
| 150 else: | |
| 151 PATCH_TOOL = path.join(BUILD_INTERNAL_DIR, 'tools', 'patch.EXE') | |
| 152 else: | |
| 153 PATCH_TOOL = '/usr/bin/patch' | |
| 154 | |
| 155 # If there is less than 100GB of disk space on the system, then we do | 88 # If there is less than 100GB of disk space on the system, then we do |
| 156 # a shallow checkout. | 89 # a shallow checkout. |
| 157 SHALLOW_CLONE_THRESHOLD = 100 * 1024 * 1024 * 1024 | 90 SHALLOW_CLONE_THRESHOLD = 100 * 1024 * 1024 * 1024 |
| 158 | 91 |
| 159 | 92 |
| 160 class SubprocessFailed(Exception): | 93 class SubprocessFailed(Exception): |
| 161 def __init__(self, message, code, output): | 94 def __init__(self, message, code, output): |
| 162 Exception.__init__(self, message) | 95 Exception.__init__(self, message) |
| 163 self.code = code | 96 self.code = code |
| 164 self.output = output | 97 self.output = output |
| (...skipping 242 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 407 except SubprocessFailed as e: | 340 except SubprocessFailed as e: |
| 408 # Throw a GclientSyncFailed exception so we can catch this independently. | 341 # Throw a GclientSyncFailed exception so we can catch this independently. |
| 409 raise GclientSyncFailed(e.message, e.code, e.output) | 342 raise GclientSyncFailed(e.message, e.code, e.output) |
| 410 else: | 343 else: |
| 411 with open(gclient_output_file) as f: | 344 with open(gclient_output_file) as f: |
| 412 return json.load(f) | 345 return json.load(f) |
| 413 finally: | 346 finally: |
| 414 os.remove(gclient_output_file) | 347 os.remove(gclient_output_file) |
| 415 | 348 |
| 416 | 349 |
| 417 def gclient_runhooks(gyp_envs): | |
| 418 gclient_bin = 'gclient.bat' if sys.platform.startswith('win') else 'gclient' | |
| 419 env = dict([env_var.split('=', 1) for env_var in gyp_envs]) | |
| 420 call(gclient_bin, 'runhooks', env=env) | |
| 421 | |
| 422 | |
| 423 def gclient_revinfo(): | 350 def gclient_revinfo(): |
| 424 gclient_bin = 'gclient.bat' if sys.platform.startswith('win') else 'gclient' | 351 gclient_bin = 'gclient.bat' if sys.platform.startswith('win') else 'gclient' |
| 425 return call(gclient_bin, 'revinfo', '-a') or '' | 352 return call(gclient_bin, 'revinfo', '-a') or '' |
| 426 | 353 |
| 427 | 354 |
| 428 def create_manifest(): | 355 def create_manifest(): |
| 429 manifest = {} | 356 manifest = {} |
| 430 output = gclient_revinfo() | 357 output = gclient_revinfo() |
| 431 for line in output.strip().splitlines(): | 358 for line in output.strip().splitlines(): |
| 432 match = REVINFO_RE.match(line.strip()) | 359 match = REVINFO_RE.match(line.strip()) |
| (...skipping 341 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 774 continue | 701 continue |
| 775 revision = get_target_revision(deps_name, deps_data.get('url', None), | 702 revision = get_target_revision(deps_name, deps_data.get('url', None), |
| 776 revisions) | 703 revisions) |
| 777 if not revision: | 704 if not revision: |
| 778 continue | 705 continue |
| 779 git('fetch', 'origin', cwd=deps_name) | 706 git('fetch', 'origin', cwd=deps_name) |
| 780 force_revision(deps_name, revision) | 707 force_revision(deps_name, revision) |
| 781 | 708 |
| 782 | 709 |
| 783 def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only, | 710 def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only, |
| 784 patch_root, issue, patchset, rietveld_server, | 711 patch_root, issue, patchset, rietveld_server, gerrit_repo, |
| 785 gerrit_repo, gerrit_ref, gerrit_rebase_patch_ref, | 712 gerrit_ref, gerrit_rebase_patch_ref, revision_mapping, |
| 786 revision_mapping, apply_issue_email_file, | 713 apply_issue_email_file, apply_issue_key_file, shallow, refs, |
| 787 apply_issue_key_file, gyp_env, shallow, runhooks, | 714 git_cache_dir, gerrit_reset): |
| 788 refs, git_cache_dir, gerrit_reset): | |
| 789 # Get a checkout of each solution, without DEPS or hooks. | 715 # Get a checkout of each solution, without DEPS or hooks. |
| 790 # Calling git directly because there is no way to run Gclient without | 716 # Calling git directly because there is no way to run Gclient without |
| 791 # invoking DEPS. | 717 # invoking DEPS. |
| 792 print 'Fetching Git checkout' | 718 print 'Fetching Git checkout' |
| 793 | 719 |
| 794 git_ref = git_checkout(solutions, revisions, shallow, refs, git_cache_dir) | 720 git_ref = git_checkout(solutions, revisions, shallow, refs, git_cache_dir) |
| 795 | 721 |
| 796 print '===Processing patch solutions===' | 722 print '===Processing patch solutions===' |
| 797 already_patched = [] | 723 already_patched = [] |
| 798 patch_root = patch_root or '' | 724 patch_root = patch_root or '' |
| (...skipping 136 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 935 parse.add_option('--output_manifest', action='store_true', | 861 parse.add_option('--output_manifest', action='store_true', |
| 936 help=('Add manifest json to the json output.')) | 862 help=('Add manifest json to the json output.')) |
| 937 parse.add_option('--slave_name', default=socket.getfqdn().split('.')[0], | 863 parse.add_option('--slave_name', default=socket.getfqdn().split('.')[0], |
| 938 help='Hostname of the current machine, ' | 864 help='Hostname of the current machine, ' |
| 939 'used for determining whether or not to activate.') | 865 'used for determining whether or not to activate.') |
| 940 parse.add_option('--build_dir', default=os.getcwd()) | 866 parse.add_option('--build_dir', default=os.getcwd()) |
| 941 parse.add_option('--flag_file', default=path.join(os.getcwd(), | 867 parse.add_option('--flag_file', default=path.join(os.getcwd(), |
| 942 'update.flag')) | 868 'update.flag')) |
| 943 parse.add_option('--shallow', action='store_true', | 869 parse.add_option('--shallow', action='store_true', |
| 944 help='Use shallow clones for cache repositories.') | 870 help='Use shallow clones for cache repositories.') |
| 945 parse.add_option('--gyp_env', action='append', default=[], | |
| 946 help='Environment variables to pass into gclient runhooks.') | |
| 947 parse.add_option('--clobber', action='store_true', | 871 parse.add_option('--clobber', action='store_true', |
| 948 help='Delete checkout first, always') | 872 help='Delete checkout first, always') |
| 949 parse.add_option('--bot_update_clobber', action='store_true', dest='clobber', | 873 parse.add_option('--bot_update_clobber', action='store_true', dest='clobber', |
| 950 help='(synonym for --clobber)') | 874 help='(synonym for --clobber)') |
| 951 parse.add_option('-o', '--output_json', | 875 parse.add_option('-o', '--output_json', |
| 952 help='Output JSON information into a specified file') | 876 help='Output JSON information into a specified file') |
| 953 parse.add_option('--no_shallow', action='store_true', | 877 parse.add_option('--no_shallow', action='store_true', |
| 954 help='Bypass disk detection and never shallow clone. ' | 878 help='Bypass disk detection and never shallow clone. ' |
| 955 'Does not override the --shallow flag') | 879 'Does not override the --shallow flag') |
| 956 parse.add_option('--no_runhooks', action='store_true', | |
| 957 help='Do not run hooks on official builder.') | |
| 958 parse.add_option('--refs', action='append', | 880 parse.add_option('--refs', action='append', |
| 959 help='Also fetch this refspec for the main solution(s). ' | 881 help='Also fetch this refspec for the main solution(s). ' |
| 960 'Eg. +refs/branch-heads/*') | 882 'Eg. +refs/branch-heads/*') |
| 961 parse.add_option('--with_branch_heads', action='store_true', | 883 parse.add_option('--with_branch_heads', action='store_true', |
| 962 help='Always pass --with_branch_heads to gclient. This ' | 884 help='Always pass --with_branch_heads to gclient. This ' |
| 963 'does the same thing as --refs +refs/branch-heads/*') | 885 'does the same thing as --refs +refs/branch-heads/*') |
| 964 parse.add_option('--git-cache-dir', default=path.join(SLAVE_DIR, 'cache_dir'), | 886 parse.add_option('--git-cache-dir', help='Path to git cache directory.') |
| 965 help='Path to git cache directory.') | |
| 966 | 887 |
| 967 | 888 |
| 968 options, args = parse.parse_args() | 889 options, args = parse.parse_args() |
| 969 | 890 |
| 891 if not options.git_cache_dir: |
| 892 parse.error('--git-cache-dir is required') |
| 893 |
| 970 if not options.refs: | 894 if not options.refs: |
| 971 options.refs = [] | 895 options.refs = [] |
| 972 | 896 |
| 973 if options.with_branch_heads: | 897 if options.with_branch_heads: |
| 974 options.refs.append(BRANCH_HEADS_REFSPEC) | 898 options.refs.append(BRANCH_HEADS_REFSPEC) |
| 975 del options.with_branch_heads | 899 del options.with_branch_heads |
| 976 | 900 |
| 977 try: | 901 try: |
| 978 if options.revision_mapping_file: | 902 if options.revision_mapping_file: |
| 979 if options.revision_mapping: | 903 if options.revision_mapping: |
| (...skipping 77 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1057 issue=options.issue, | 981 issue=options.issue, |
| 1058 patchset=options.patchset, | 982 patchset=options.patchset, |
| 1059 rietveld_server=options.rietveld_server, | 983 rietveld_server=options.rietveld_server, |
| 1060 gerrit_repo=options.gerrit_repo, | 984 gerrit_repo=options.gerrit_repo, |
| 1061 gerrit_ref=options.gerrit_ref, | 985 gerrit_ref=options.gerrit_ref, |
| 1062 gerrit_rebase_patch_ref=not options.gerrit_no_rebase_patch_ref, | 986 gerrit_rebase_patch_ref=not options.gerrit_no_rebase_patch_ref, |
| 1063 revision_mapping=options.revision_mapping, | 987 revision_mapping=options.revision_mapping, |
| 1064 apply_issue_email_file=options.apply_issue_email_file, | 988 apply_issue_email_file=options.apply_issue_email_file, |
| 1065 apply_issue_key_file=options.apply_issue_key_file, | 989 apply_issue_key_file=options.apply_issue_key_file, |
| 1066 | 990 |
| 1067 # For official builders. | |
| 1068 gyp_env=options.gyp_env, | |
| 1069 runhooks=not options.no_runhooks, | |
| 1070 | |
| 1071 # Finally, extra configurations such as shallowness of the clone. | 991 # Finally, extra configurations such as shallowness of the clone. |
| 1072 shallow=options.shallow, | 992 shallow=options.shallow, |
| 1073 refs=options.refs, | 993 refs=options.refs, |
| 1074 git_cache_dir=options.git_cache_dir, | 994 git_cache_dir=options.git_cache_dir, |
| 1075 gerrit_reset=not options.gerrit_no_reset) | 995 gerrit_reset=not options.gerrit_no_reset) |
| 1076 gclient_output = ensure_checkout(**checkout_parameters) | 996 gclient_output = ensure_checkout(**checkout_parameters) |
| 1077 except GclientSyncFailed: | 997 except GclientSyncFailed: |
| 1078 print 'We failed gclient sync, lets delete the checkout and retry.' | 998 print 'We failed gclient sync, lets delete the checkout and retry.' |
| 1079 ensure_no_checkout(dir_names) | 999 ensure_no_checkout(dir_names) |
| 1080 gclient_output = ensure_checkout(**checkout_parameters) | 1000 gclient_output = ensure_checkout(**checkout_parameters) |
| (...skipping 49 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1130 else: | 1050 else: |
| 1131 # If we're not on recipes, tell annotator about our got_revisions. | 1051 # If we're not on recipes, tell annotator about our got_revisions. |
| 1132 emit_properties(got_revisions) | 1052 emit_properties(got_revisions) |
| 1133 | 1053 |
| 1134 | 1054 |
| 1135 def print_debug_info(): | 1055 def print_debug_info(): |
| 1136 print "Debugging info:" | 1056 print "Debugging info:" |
| 1137 debug_params = { | 1057 debug_params = { |
| 1138 'CURRENT_DIR': CURRENT_DIR, | 1058 'CURRENT_DIR': CURRENT_DIR, |
| 1139 'BUILDER_DIR': BUILDER_DIR, | 1059 'BUILDER_DIR': BUILDER_DIR, |
| 1140 'SLAVE_DIR': SLAVE_DIR, | |
| 1141 'THIS_DIR': THIS_DIR, | 1060 'THIS_DIR': THIS_DIR, |
| 1142 'SCRIPTS_DIR': SCRIPTS_DIR, | |
| 1143 'BUILD_DIR': BUILD_DIR, | |
| 1144 'ROOT_DIR': ROOT_DIR, | |
| 1145 'DEPOT_TOOLS_DIR': DEPOT_TOOLS_DIR, | 1061 'DEPOT_TOOLS_DIR': DEPOT_TOOLS_DIR, |
| 1146 } | 1062 } |
| 1147 for k, v in sorted(debug_params.iteritems()): | 1063 for k, v in sorted(debug_params.iteritems()): |
| 1148 print "%s: %r" % (k, v) | 1064 print "%s: %r" % (k, v) |
| 1149 | 1065 |
| 1150 | 1066 |
| 1151 def main(): | 1067 def main(): |
| 1152 # Get inputs. | 1068 # Get inputs. |
| 1153 options, _ = parse_args() | 1069 options, _ = parse_args() |
| 1154 | 1070 |
| (...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1190 except Exception: | 1106 except Exception: |
| 1191 # Unexpected failure. | 1107 # Unexpected failure. |
| 1192 emit_flag(options.flag_file) | 1108 emit_flag(options.flag_file) |
| 1193 raise | 1109 raise |
| 1194 else: | 1110 else: |
| 1195 emit_flag(options.flag_file) | 1111 emit_flag(options.flag_file) |
| 1196 | 1112 |
| 1197 | 1113 |
| 1198 if __name__ == '__main__': | 1114 if __name__ == '__main__': |
| 1199 sys.exit(main()) | 1115 sys.exit(main()) |
| OLD | NEW |