Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(555)

Side by Side Diff: recipe_modules/bot_update/resources/bot_update.py

Issue 2328883002: Remove unused PATCH_TOOL and resulting dead code from bot_update. (Closed)
Patch Set: Fix "test" Created 4 years, 3 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | tests/bot_update_coverage_test.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # Copyright 2014 The Chromium Authors. All rights reserved. 2 # Copyright 2014 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 5
6 # TODO(hinoka): Use logging. 6 # TODO(hinoka): Use logging.
7 7
8 import cStringIO 8 import cStringIO
9 import codecs 9 import codecs
10 import collections 10 import collections
(...skipping 14 matching lines...) Expand all
25 import urllib2 25 import urllib2
26 import urlparse 26 import urlparse
27 import uuid 27 import uuid
28 28
29 import os.path as path 29 import os.path as path
30 30
31 # How many bytes at a time to read from pipes. 31 # How many bytes at a time to read from pipes.
32 BUF_SIZE = 256 32 BUF_SIZE = 256
33 33
34 34
35 # TODO(luqui): This is a horrible hack to identify build_internal when build
36 # is a recipe dependency. bot_update should not be depending on internal,
37 # rather the arrow should go the other way (or just be destroyed).
38 def check_dir(name, dirs, default=None):
39 for d in dirs:
40 d = path.abspath(d)
41 if path.basename(d) == name and path.isdir(d):
42 return d
43 return default
44
45
46 # Define a bunch of directory paths. 35 # Define a bunch of directory paths.
47 # Relative to the current working directory. 36 # Relative to the current working directory.
48 CURRENT_DIR = path.abspath(os.getcwd()) 37 CURRENT_DIR = path.abspath(os.getcwd())
49 BUILDER_DIR = path.dirname(CURRENT_DIR) 38 BUILDER_DIR = path.dirname(CURRENT_DIR)
50 SLAVE_DIR = path.dirname(BUILDER_DIR)
51 39
52 # Relative to this script's filesystem path. 40 # Relative to this script's filesystem path.
53 THIS_DIR = path.dirname(path.abspath(__file__)) 41 THIS_DIR = path.dirname(path.abspath(__file__))
54 SCRIPTS_DIR = check_dir(
55 'scripts', [
56 path.dirname(THIS_DIR),
57 path.join(SLAVE_DIR, '..', 'scripts'),
58 path.join(THIS_DIR, # resources
59 '..', # bot_update
60 '..', # recipe_modules
61 '..', # depot_tools
62 '..', # .recipe_deps
63 '..', # slave
64 '..', # scripts
65 '..', # build_internal
66 '..', # ROOT_DIR
67 'build',
68 'scripts'),
69 path.join(SLAVE_DIR, '..', 'build', 'scripts'),
70 ], default=path.dirname(THIS_DIR))
71 BUILD_DIR = path.dirname(SCRIPTS_DIR)
72 ROOT_DIR = path.dirname(BUILD_DIR)
73 42
74 DEPOT_TOOLS_DIR = path.abspath(path.join(THIS_DIR, '..', '..', '..')) 43 DEPOT_TOOLS_DIR = path.abspath(path.join(THIS_DIR, '..', '..', '..'))
75 44
76 BUILD_INTERNAL_DIR = check_dir(
77 'build_internal', [
78 path.join(ROOT_DIR, 'build_internal'),
79 path.join(ROOT_DIR, # .recipe_deps
80 path.pardir, # slave
81 path.pardir, # scripts
82 path.pardir), # build_internal
83 ])
84
85
86 CHROMIUM_GIT_HOST = 'https://chromium.googlesource.com' 45 CHROMIUM_GIT_HOST = 'https://chromium.googlesource.com'
87 CHROMIUM_SRC_URL = CHROMIUM_GIT_HOST + '/chromium/src.git' 46 CHROMIUM_SRC_URL = CHROMIUM_GIT_HOST + '/chromium/src.git'
88 47
89 BRANCH_HEADS_REFSPEC = '+refs/branch-heads/*' 48 BRANCH_HEADS_REFSPEC = '+refs/branch-heads/*'
90 49
91 # Regular expression that matches a single commit footer line. 50 # Regular expression that matches a single commit footer line.
92 COMMIT_FOOTER_ENTRY_RE = re.compile(r'([^:]+):\s+(.+)') 51 COMMIT_FOOTER_ENTRY_RE = re.compile(r'([^:]+):\s+(.+)')
93 52
94 # Footer metadata keys for regular and gsubtreed mirrored commit positions. 53 # Footer metadata keys for regular and gsubtreed mirrored commit positions.
95 COMMIT_POSITION_FOOTER_KEY = 'Cr-Commit-Position' 54 COMMIT_POSITION_FOOTER_KEY = 'Cr-Commit-Position'
(...skipping 18 matching lines...) Expand all
114 73
115 74
116 GCLIENT_TEMPLATE = """solutions = %(solutions)s 75 GCLIENT_TEMPLATE = """solutions = %(solutions)s
117 76
118 cache_dir = r%(cache_dir)s 77 cache_dir = r%(cache_dir)s
119 %(target_os)s 78 %(target_os)s
120 %(target_os_only)s 79 %(target_os_only)s
121 """ 80 """
122 81
123 82
124 internal_data = {}
125 if BUILD_INTERNAL_DIR:
126 local_vars = {}
127 try:
128 execfile(os.path.join(
129 BUILD_INTERNAL_DIR, 'scripts', 'slave', 'bot_update_cfg.py'),
130 local_vars)
131 except Exception:
132 # Same as if BUILD_INTERNAL_DIR didn't exist in the first place.
133 print 'Warning: unable to read internal configuration file.'
134 print 'If this is an internal bot, this step may be erroneously inactive.'
135 internal_data = local_vars
136
137
138
139 # How many times to try before giving up. 83 # How many times to try before giving up.
140 ATTEMPTS = 5 84 ATTEMPTS = 5
141 85
142 GIT_CACHE_PATH = path.join(DEPOT_TOOLS_DIR, 'git_cache.py') 86 GIT_CACHE_PATH = path.join(DEPOT_TOOLS_DIR, 'git_cache.py')
143 87
144 # Find the patch tool.
145 if sys.platform.startswith('win'):
146 if not BUILD_INTERNAL_DIR:
147 print 'Warning: could not find patch tool because there is no '
148 print 'build_internal present.'
149 PATCH_TOOL = None
150 else:
151 PATCH_TOOL = path.join(BUILD_INTERNAL_DIR, 'tools', 'patch.EXE')
152 else:
153 PATCH_TOOL = '/usr/bin/patch'
154
155 # If there is less than 100GB of disk space on the system, then we do 88 # If there is less than 100GB of disk space on the system, then we do
156 # a shallow checkout. 89 # a shallow checkout.
157 SHALLOW_CLONE_THRESHOLD = 100 * 1024 * 1024 * 1024 90 SHALLOW_CLONE_THRESHOLD = 100 * 1024 * 1024 * 1024
158 91
159 92
160 class SubprocessFailed(Exception): 93 class SubprocessFailed(Exception):
161 def __init__(self, message, code, output): 94 def __init__(self, message, code, output):
162 Exception.__init__(self, message) 95 Exception.__init__(self, message)
163 self.code = code 96 self.code = code
164 self.output = output 97 self.output = output
(...skipping 242 matching lines...) Expand 10 before | Expand all | Expand 10 after
407 except SubprocessFailed as e: 340 except SubprocessFailed as e:
408 # Throw a GclientSyncFailed exception so we can catch this independently. 341 # Throw a GclientSyncFailed exception so we can catch this independently.
409 raise GclientSyncFailed(e.message, e.code, e.output) 342 raise GclientSyncFailed(e.message, e.code, e.output)
410 else: 343 else:
411 with open(gclient_output_file) as f: 344 with open(gclient_output_file) as f:
412 return json.load(f) 345 return json.load(f)
413 finally: 346 finally:
414 os.remove(gclient_output_file) 347 os.remove(gclient_output_file)
415 348
416 349
417 def gclient_runhooks(gyp_envs):
418 gclient_bin = 'gclient.bat' if sys.platform.startswith('win') else 'gclient'
419 env = dict([env_var.split('=', 1) for env_var in gyp_envs])
420 call(gclient_bin, 'runhooks', env=env)
421
422
423 def gclient_revinfo(): 350 def gclient_revinfo():
424 gclient_bin = 'gclient.bat' if sys.platform.startswith('win') else 'gclient' 351 gclient_bin = 'gclient.bat' if sys.platform.startswith('win') else 'gclient'
425 return call(gclient_bin, 'revinfo', '-a') or '' 352 return call(gclient_bin, 'revinfo', '-a') or ''
426 353
427 354
428 def create_manifest(): 355 def create_manifest():
429 manifest = {} 356 manifest = {}
430 output = gclient_revinfo() 357 output = gclient_revinfo()
431 for line in output.strip().splitlines(): 358 for line in output.strip().splitlines():
432 match = REVINFO_RE.match(line.strip()) 359 match = REVINFO_RE.match(line.strip())
(...skipping 341 matching lines...) Expand 10 before | Expand all | Expand 10 after
774 continue 701 continue
775 revision = get_target_revision(deps_name, deps_data.get('url', None), 702 revision = get_target_revision(deps_name, deps_data.get('url', None),
776 revisions) 703 revisions)
777 if not revision: 704 if not revision:
778 continue 705 continue
779 git('fetch', 'origin', cwd=deps_name) 706 git('fetch', 'origin', cwd=deps_name)
780 force_revision(deps_name, revision) 707 force_revision(deps_name, revision)
781 708
782 709
783 def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only, 710 def ensure_checkout(solutions, revisions, first_sln, target_os, target_os_only,
784 patch_root, issue, patchset, rietveld_server, 711 patch_root, issue, patchset, rietveld_server, gerrit_repo,
785 gerrit_repo, gerrit_ref, gerrit_rebase_patch_ref, 712 gerrit_ref, gerrit_rebase_patch_ref, revision_mapping,
786 revision_mapping, apply_issue_email_file, 713 apply_issue_email_file, apply_issue_key_file, shallow, refs,
787 apply_issue_key_file, gyp_env, shallow, runhooks, 714 git_cache_dir, gerrit_reset):
788 refs, git_cache_dir, gerrit_reset):
789 # Get a checkout of each solution, without DEPS or hooks. 715 # Get a checkout of each solution, without DEPS or hooks.
790 # Calling git directly because there is no way to run Gclient without 716 # Calling git directly because there is no way to run Gclient without
791 # invoking DEPS. 717 # invoking DEPS.
792 print 'Fetching Git checkout' 718 print 'Fetching Git checkout'
793 719
794 git_ref = git_checkout(solutions, revisions, shallow, refs, git_cache_dir) 720 git_ref = git_checkout(solutions, revisions, shallow, refs, git_cache_dir)
795 721
796 print '===Processing patch solutions===' 722 print '===Processing patch solutions==='
797 already_patched = [] 723 already_patched = []
798 patch_root = patch_root or '' 724 patch_root = patch_root or ''
(...skipping 136 matching lines...) Expand 10 before | Expand all | Expand 10 after
935 parse.add_option('--output_manifest', action='store_true', 861 parse.add_option('--output_manifest', action='store_true',
936 help=('Add manifest json to the json output.')) 862 help=('Add manifest json to the json output.'))
937 parse.add_option('--slave_name', default=socket.getfqdn().split('.')[0], 863 parse.add_option('--slave_name', default=socket.getfqdn().split('.')[0],
938 help='Hostname of the current machine, ' 864 help='Hostname of the current machine, '
939 'used for determining whether or not to activate.') 865 'used for determining whether or not to activate.')
940 parse.add_option('--build_dir', default=os.getcwd()) 866 parse.add_option('--build_dir', default=os.getcwd())
941 parse.add_option('--flag_file', default=path.join(os.getcwd(), 867 parse.add_option('--flag_file', default=path.join(os.getcwd(),
942 'update.flag')) 868 'update.flag'))
943 parse.add_option('--shallow', action='store_true', 869 parse.add_option('--shallow', action='store_true',
944 help='Use shallow clones for cache repositories.') 870 help='Use shallow clones for cache repositories.')
945 parse.add_option('--gyp_env', action='append', default=[],
946 help='Environment variables to pass into gclient runhooks.')
947 parse.add_option('--clobber', action='store_true', 871 parse.add_option('--clobber', action='store_true',
948 help='Delete checkout first, always') 872 help='Delete checkout first, always')
949 parse.add_option('--bot_update_clobber', action='store_true', dest='clobber', 873 parse.add_option('--bot_update_clobber', action='store_true', dest='clobber',
950 help='(synonym for --clobber)') 874 help='(synonym for --clobber)')
951 parse.add_option('-o', '--output_json', 875 parse.add_option('-o', '--output_json',
952 help='Output JSON information into a specified file') 876 help='Output JSON information into a specified file')
953 parse.add_option('--no_shallow', action='store_true', 877 parse.add_option('--no_shallow', action='store_true',
954 help='Bypass disk detection and never shallow clone. ' 878 help='Bypass disk detection and never shallow clone. '
955 'Does not override the --shallow flag') 879 'Does not override the --shallow flag')
956 parse.add_option('--no_runhooks', action='store_true',
957 help='Do not run hooks on official builder.')
958 parse.add_option('--refs', action='append', 880 parse.add_option('--refs', action='append',
959 help='Also fetch this refspec for the main solution(s). ' 881 help='Also fetch this refspec for the main solution(s). '
960 'Eg. +refs/branch-heads/*') 882 'Eg. +refs/branch-heads/*')
961 parse.add_option('--with_branch_heads', action='store_true', 883 parse.add_option('--with_branch_heads', action='store_true',
962 help='Always pass --with_branch_heads to gclient. This ' 884 help='Always pass --with_branch_heads to gclient. This '
963 'does the same thing as --refs +refs/branch-heads/*') 885 'does the same thing as --refs +refs/branch-heads/*')
964 parse.add_option('--git-cache-dir', default=path.join(SLAVE_DIR, 'cache_dir'), 886 parse.add_option('--git-cache-dir', help='Path to git cache directory.')
965 help='Path to git cache directory.')
966 887
967 888
968 options, args = parse.parse_args() 889 options, args = parse.parse_args()
969 890
891 if not options.git_cache_dir:
892 parse.error('--git-cache-dir is required')
893
970 if not options.refs: 894 if not options.refs:
971 options.refs = [] 895 options.refs = []
972 896
973 if options.with_branch_heads: 897 if options.with_branch_heads:
974 options.refs.append(BRANCH_HEADS_REFSPEC) 898 options.refs.append(BRANCH_HEADS_REFSPEC)
975 del options.with_branch_heads 899 del options.with_branch_heads
976 900
977 try: 901 try:
978 if options.revision_mapping_file: 902 if options.revision_mapping_file:
979 if options.revision_mapping: 903 if options.revision_mapping:
(...skipping 77 matching lines...) Expand 10 before | Expand all | Expand 10 after
1057 issue=options.issue, 981 issue=options.issue,
1058 patchset=options.patchset, 982 patchset=options.patchset,
1059 rietveld_server=options.rietveld_server, 983 rietveld_server=options.rietveld_server,
1060 gerrit_repo=options.gerrit_repo, 984 gerrit_repo=options.gerrit_repo,
1061 gerrit_ref=options.gerrit_ref, 985 gerrit_ref=options.gerrit_ref,
1062 gerrit_rebase_patch_ref=not options.gerrit_no_rebase_patch_ref, 986 gerrit_rebase_patch_ref=not options.gerrit_no_rebase_patch_ref,
1063 revision_mapping=options.revision_mapping, 987 revision_mapping=options.revision_mapping,
1064 apply_issue_email_file=options.apply_issue_email_file, 988 apply_issue_email_file=options.apply_issue_email_file,
1065 apply_issue_key_file=options.apply_issue_key_file, 989 apply_issue_key_file=options.apply_issue_key_file,
1066 990
1067 # For official builders.
1068 gyp_env=options.gyp_env,
1069 runhooks=not options.no_runhooks,
1070
1071 # Finally, extra configurations such as shallowness of the clone. 991 # Finally, extra configurations such as shallowness of the clone.
1072 shallow=options.shallow, 992 shallow=options.shallow,
1073 refs=options.refs, 993 refs=options.refs,
1074 git_cache_dir=options.git_cache_dir, 994 git_cache_dir=options.git_cache_dir,
1075 gerrit_reset=not options.gerrit_no_reset) 995 gerrit_reset=not options.gerrit_no_reset)
1076 gclient_output = ensure_checkout(**checkout_parameters) 996 gclient_output = ensure_checkout(**checkout_parameters)
1077 except GclientSyncFailed: 997 except GclientSyncFailed:
1078 print 'We failed gclient sync, lets delete the checkout and retry.' 998 print 'We failed gclient sync, lets delete the checkout and retry.'
1079 ensure_no_checkout(dir_names) 999 ensure_no_checkout(dir_names)
1080 gclient_output = ensure_checkout(**checkout_parameters) 1000 gclient_output = ensure_checkout(**checkout_parameters)
(...skipping 49 matching lines...) Expand 10 before | Expand all | Expand 10 after
1130 else: 1050 else:
1131 # If we're not on recipes, tell annotator about our got_revisions. 1051 # If we're not on recipes, tell annotator about our got_revisions.
1132 emit_properties(got_revisions) 1052 emit_properties(got_revisions)
1133 1053
1134 1054
1135 def print_debug_info(): 1055 def print_debug_info():
1136 print "Debugging info:" 1056 print "Debugging info:"
1137 debug_params = { 1057 debug_params = {
1138 'CURRENT_DIR': CURRENT_DIR, 1058 'CURRENT_DIR': CURRENT_DIR,
1139 'BUILDER_DIR': BUILDER_DIR, 1059 'BUILDER_DIR': BUILDER_DIR,
1140 'SLAVE_DIR': SLAVE_DIR,
1141 'THIS_DIR': THIS_DIR, 1060 'THIS_DIR': THIS_DIR,
1142 'SCRIPTS_DIR': SCRIPTS_DIR,
1143 'BUILD_DIR': BUILD_DIR,
1144 'ROOT_DIR': ROOT_DIR,
1145 'DEPOT_TOOLS_DIR': DEPOT_TOOLS_DIR, 1061 'DEPOT_TOOLS_DIR': DEPOT_TOOLS_DIR,
1146 } 1062 }
1147 for k, v in sorted(debug_params.iteritems()): 1063 for k, v in sorted(debug_params.iteritems()):
1148 print "%s: %r" % (k, v) 1064 print "%s: %r" % (k, v)
1149 1065
1150 1066
1151 def main(): 1067 def main():
1152 # Get inputs. 1068 # Get inputs.
1153 options, _ = parse_args() 1069 options, _ = parse_args()
1154 1070
(...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after
1190 except Exception: 1106 except Exception:
1191 # Unexpected failure. 1107 # Unexpected failure.
1192 emit_flag(options.flag_file) 1108 emit_flag(options.flag_file)
1193 raise 1109 raise
1194 else: 1110 else:
1195 emit_flag(options.flag_file) 1111 emit_flag(options.flag_file)
1196 1112
1197 1113
1198 if __name__ == '__main__': 1114 if __name__ == '__main__':
1199 sys.exit(main()) 1115 sys.exit(main())
OLDNEW
« no previous file with comments | « no previous file | tests/bot_update_coverage_test.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698