Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(345)

Side by Side Diff: fetch.py

Issue 106403003: fetch.py: Use '--nohooks' when calling 'gclient sync' (Closed) Base URL: https://chromium.googlesource.com/chromium/tools/depot_tools.git@master
Patch Set: Address Dirk's notes. Created 7 years ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. 2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 5
6 """ 6 """
7 Tool to perform checkouts in one easy command line! 7 Tool to perform checkouts in one easy command line!
8 8
9 Usage: 9 Usage:
10 fetch <recipe> [--property=value [--property2=value2 ...]] 10 fetch <recipe> [--property=value [--property2=value2 ...]]
11 11
12 This script is a wrapper around various version control and repository 12 This script is a wrapper around various version control and repository
13 checkout commands. It requires a |recipe| name, fetches data from that 13 checkout commands. It requires a |recipe| name, fetches data from that
14 recipe in depot_tools/recipes, and then performs all necessary inits, 14 recipe in depot_tools/recipes, and then performs all necessary inits,
15 checkouts, pulls, fetches, etc. 15 checkouts, pulls, fetches, etc.
16 16
17 Optional arguments may be passed on the command line in key-value pairs. 17 Optional arguments may be passed on the command line in key-value pairs.
18 These parameters will be passed through to the recipe's main method. 18 These parameters will be passed through to the recipe's main method.
19 """ 19 """
20 20
21 import json 21 import json
22 import optparse
22 import os 23 import os
23 import subprocess 24 import subprocess
24 import sys 25 import sys
25 import pipes 26 import pipes
26 27
27 from distutils import spawn 28 from distutils import spawn
28 29
29 30
30 SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__)) 31 SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__))
31 32
32
33 ################################################# 33 #################################################
34 # Checkout class definitions. 34 # Checkout class definitions.
35 ################################################# 35 #################################################
36 class Checkout(object): 36 class Checkout(object):
37 """Base class for implementing different types of checkouts. 37 """Base class for implementing different types of checkouts.
38 38
39 Attributes: 39 Attributes:
40 |base|: the absolute path of the directory in which this script is run. 40 |base|: the absolute path of the directory in which this script is run.
41 |spec|: the spec for this checkout as returned by the recipe. Different 41 |spec|: the spec for this checkout as returned by the recipe. Different
42 subclasses will expect different keys in this dictionary. 42 subclasses will expect different keys in this dictionary.
43 |root|: the directory into which the checkout will be performed, as returned 43 |root|: the directory into which the checkout will be performed, as returned
44 by the recipe. This is a relative path from |base|. 44 by the recipe. This is a relative path from |base|.
45 """ 45 """
46 def __init__(self, dryrun, spec, root): 46 def __init__(self, options, spec, root):
47 self.base = os.getcwd() 47 self.base = os.getcwd()
48 self.dryrun = dryrun 48 self.options = options
49 self.spec = spec 49 self.spec = spec
50 self.root = root 50 self.root = root
51 51
52 def exists(self): 52 def exists(self):
53 pass 53 pass
54 54
55 def init(self): 55 def init(self):
56 pass 56 pass
57 57
58 def sync(self): 58 def sync(self):
59 pass 59 pass
60 60
61 def run(self, cmd, **kwargs): 61 def run(self, cmd, **kwargs):
62 print 'Running: %s' % (' '.join(pipes.quote(x) for x in cmd)) 62 print 'Running: %s' % (' '.join(pipes.quote(x) for x in cmd))
63 if self.dryrun: 63 if self.options.dryrun:
64 return 0 64 return 0
65 return subprocess.check_call(cmd, **kwargs) 65 return subprocess.check_call(cmd, **kwargs)
66 66
67 67
68 class GclientCheckout(Checkout): 68 class GclientCheckout(Checkout):
69 69
70 def run_gclient(self, *cmd, **kwargs): 70 def run_gclient(self, *cmd, **kwargs):
71 if not spawn.find_executable('gclient'): 71 if not spawn.find_executable('gclient'):
72 cmd_prefix = (sys.executable, os.path.join(SCRIPT_PATH, 'gclient.py')) 72 cmd_prefix = (sys.executable, os.path.join(SCRIPT_PATH, 'gclient.py'))
73 else: 73 else:
(...skipping 16 matching lines...) Expand all
90 def run_svn(self, *cmd, **kwargs): 90 def run_svn(self, *cmd, **kwargs):
91 if sys.platform == 'win32' and not spawn.find_executable('svn'): 91 if sys.platform == 'win32' and not spawn.find_executable('svn'):
92 svn_path = os.path.join(SCRIPT_PATH, 'svn_bin', 'svn.exe') 92 svn_path = os.path.join(SCRIPT_PATH, 'svn_bin', 'svn.exe')
93 else: 93 else:
94 svn_path = 'svn' 94 svn_path = 'svn'
95 return self.run((svn_path,) + cmd, **kwargs) 95 return self.run((svn_path,) + cmd, **kwargs)
96 96
97 97
98 class GclientGitCheckout(GclientCheckout, GitCheckout): 98 class GclientGitCheckout(GclientCheckout, GitCheckout):
99 99
100 def __init__(self, dryrun, spec, root): 100 def __init__(self, options, spec, root):
101 super(GclientGitCheckout, self).__init__(dryrun, spec, root) 101 super(GclientGitCheckout, self).__init__(options, spec, root)
102 assert 'solutions' in self.spec 102 assert 'solutions' in self.spec
103 keys = ['solutions', 'target_os', 'target_os_only'] 103 keys = ['solutions', 'target_os', 'target_os_only']
104 gclient_spec = '\n'.join('%s = %s' % (key, self.spec[key]) 104 gclient_spec = '\n'.join('%s = %s' % (key, self.spec[key])
105 for key in keys if key in self.spec) 105 for key in keys if key in self.spec)
106 self.spec['gclient_spec'] = gclient_spec 106 self.spec['gclient_spec'] = gclient_spec
107 107
108 def exists(self): 108 def exists(self):
109 return os.path.exists(os.path.join(os.getcwd(), self.root)) 109 return os.path.exists(os.path.join(os.getcwd(), self.root))
110 110
111 def init(self): 111 def init(self):
112 # TODO(dpranke): Work around issues w/ delta compression on big repos. 112 # TODO(dpranke): Work around issues w/ delta compression on big repos.
113 self.run_git('config', '--global', 'core.deltaBaseCacheLimit', '1G') 113 self.run_git('config', '--global', 'core.deltaBaseCacheLimit', '1G')
114 114
115 # Configure and do the gclient checkout. 115 # Configure and do the gclient checkout.
116 self.run_gclient('config', '--spec', self.spec['gclient_spec']) 116 self.run_gclient('config', '--spec', self.spec['gclient_spec'])
117 self.run_gclient('sync') 117 if self.options.nohooks:
118 self.run_gclient('sync', '--nohooks')
119 else:
120 self.run_gclient('sync')
118 121
119 # Configure git. 122 # Configure git.
120 wd = os.path.join(self.base, self.root) 123 wd = os.path.join(self.base, self.root)
121 if self.dryrun: 124 if self.options.dryrun:
122 print 'cd %s' % wd 125 print 'cd %s' % wd
123 self.run_git( 126 self.run_git(
124 'submodule', 'foreach', 127 'submodule', 'foreach',
125 'git config -f $toplevel/.git/config submodule.$name.ignore all', 128 'git config -f $toplevel/.git/config submodule.$name.ignore all',
126 cwd=wd) 129 cwd=wd)
127 self.run_git('config', 'diff.ignoreSubmodules', 'all', cwd=wd) 130 self.run_git('config', 'diff.ignoreSubmodules', 'all', cwd=wd)
128 131
129 132
130 class GclientGitSvnCheckout(GclientGitCheckout, SvnCheckout): 133 class GclientGitSvnCheckout(GclientGitCheckout, SvnCheckout):
131 134
132 def __init__(self, dryrun, spec, root): 135 def __init__(self, options, spec, root):
133 super(GclientGitSvnCheckout, self).__init__(dryrun, spec, root) 136 super(GclientGitSvnCheckout, self).__init__(options, spec, root)
134 assert 'svn_url' in self.spec 137 assert 'svn_url' in self.spec
135 assert 'svn_branch' in self.spec 138 assert 'svn_branch' in self.spec
136 assert 'svn_ref' in self.spec 139 assert 'svn_ref' in self.spec
137 140
138 def init(self): 141 def init(self):
139 # Ensure we are authenticated with subversion for all submodules. 142 # Ensure we are authenticated with subversion for all submodules.
140 git_svn_dirs = json.loads(self.spec.get('submodule_git_svn_spec', '{}')) 143 git_svn_dirs = json.loads(self.spec.get('submodule_git_svn_spec', '{}'))
141 git_svn_dirs.update({self.root: self.spec}) 144 git_svn_dirs.update({self.root: self.spec})
142 for _, svn_spec in git_svn_dirs.iteritems(): 145 for _, svn_spec in git_svn_dirs.iteritems():
143 try: 146 try:
144 self.run_svn('ls', '--non-interactive', svn_spec['svn_url']) 147 self.run_svn('ls', '--non-interactive', svn_spec['svn_url'])
145 except subprocess.CalledProcessError: 148 except subprocess.CalledProcessError:
146 print 'Please run `svn ls %s`' % svn_spec['svn_url'] 149 print 'Please run `svn ls %s`' % svn_spec['svn_url']
147 return 1 150 return 1
148 151
149 super(GclientGitSvnCheckout, self).init() 152 super(GclientGitSvnCheckout, self).init()
150 153
151 # Configure git-svn. 154 # Configure git-svn.
152 for path, svn_spec in git_svn_dirs.iteritems(): 155 for path, svn_spec in git_svn_dirs.iteritems():
153 real_path = os.path.join(*path.split('/')) 156 real_path = os.path.join(*path.split('/'))
154 if real_path != self.root: 157 if real_path != self.root:
155 real_path = os.path.join(self.root, real_path) 158 real_path = os.path.join(self.root, real_path)
156 wd = os.path.join(self.base, real_path) 159 wd = os.path.join(self.base, real_path)
157 if self.dryrun: 160 if self.options.dryrun:
158 print 'cd %s' % wd 161 print 'cd %s' % wd
159 self.run_git('svn', 'init', '--prefix=origin/', '-T', 162 self.run_git('svn', 'init', '--prefix=origin/', '-T',
160 svn_spec['svn_branch'], svn_spec['svn_url'], cwd=wd) 163 svn_spec['svn_branch'], svn_spec['svn_url'], cwd=wd)
161 self.run_git('config', '--replace', 'svn-remote.svn.fetch', 164 self.run_git('config', '--replace', 'svn-remote.svn.fetch',
162 svn_spec['svn_branch'] + ':refs/remotes/origin/' + 165 svn_spec['svn_branch'] + ':refs/remotes/origin/' +
163 svn_spec['svn_ref'], cwd=wd) 166 svn_spec['svn_ref'], cwd=wd)
164 self.run_git('svn', 'fetch', cwd=wd) 167 self.run_git('svn', 'fetch', cwd=wd)
165 168
166 169
167 170
168 CHECKOUT_TYPE_MAP = { 171 CHECKOUT_TYPE_MAP = {
169 'gclient': GclientCheckout, 172 'gclient': GclientCheckout,
170 'gclient_git': GclientGitCheckout, 173 'gclient_git': GclientGitCheckout,
171 'gclient_git_svn': GclientGitSvnCheckout, 174 'gclient_git_svn': GclientGitSvnCheckout,
172 'git': GitCheckout, 175 'git': GitCheckout,
173 } 176 }
174 177
175 178
176 def CheckoutFactory(type_name, dryrun, spec, root): 179 def CheckoutFactory(type_name, options, spec, root):
177 """Factory to build Checkout class instances.""" 180 """Factory to build Checkout class instances."""
178 class_ = CHECKOUT_TYPE_MAP.get(type_name) 181 class_ = CHECKOUT_TYPE_MAP.get(type_name)
179 if not class_: 182 if not class_:
180 raise KeyError('unrecognized checkout type: %s' % type_name) 183 raise KeyError('unrecognized checkout type: %s' % type_name)
181 return class_(dryrun, spec, root) 184 return class_(options, spec, root)
182 185
183 186
184 ################################################# 187 #################################################
185 # Utility function and file entry point. 188 # Utility function and file entry point.
186 ################################################# 189 #################################################
187 def usage(msg=None): 190 def usage(msg=None):
188 """Print help and exit.""" 191 """Print help and exit."""
189 if msg: 192 if msg:
190 print 'Error:', msg 193 print 'Error:', msg
191 194
192 print ( 195 print (
193 """ 196 """
194 usage: %s [-n|--dry-run] <recipe> [--property=value [--property2=value2 ...]] 197 usage: %s [options] <recipe> [--property=value [--property2=value2 ...]]
198
199 This script can be used to download the Chromium sources. See
200 http://www.chromium.org/developers/how-tos/get-the-code
201 for full usage instructions.
202
203 Valid options:
204 -h, --help, help Print this message.
205 --nohooks Don't run hooks after checkout.
206 -n, --dryrun Don't run commands, only print them.
195 """ % os.path.basename(sys.argv[0])) 207 """ % os.path.basename(sys.argv[0]))
196 sys.exit(bool(msg)) 208 sys.exit(bool(msg))
197 209
198 210
199 def handle_args(argv): 211 def handle_args(argv):
200 """Gets the recipe name from the command line arguments.""" 212 """Gets the recipe name from the command line arguments."""
201 if len(argv) <= 1: 213 if len(argv) <= 1:
202 usage('Must specify a recipe.') 214 usage('Must specify a recipe.')
203 if argv[1] in ('-h', '--help', 'help'): 215 if argv[1] in ('-h', '--help', 'help'):
204 usage() 216 usage()
205 217
206 dryrun = False 218 dryrun = False
207 if argv[1] in ('-n', '--dry-run'): 219 nohooks = False
208 dryrun = True 220 while len(argv) >= 2:
221 arg = argv[1]
222 if not arg.startswith('-'):
223 break
209 argv.pop(1) 224 argv.pop(1)
225 if arg in ('-n', '--dry-run'):
226 dryrun = True
227 elif arg == '--nohooks':
228 nohooks = True
229 else:
230 usage('Invalid option %s.' % arg)
210 231
211 def looks_like_arg(arg): 232 def looks_like_arg(arg):
212 return arg.startswith('--') and arg.count('=') == 1 233 return arg.startswith('--') and arg.count('=') == 1
213 234
214 bad_parms = [x for x in argv[2:] if not looks_like_arg(x)] 235 bad_parms = [x for x in argv[2:] if not looks_like_arg(x)]
215 if bad_parms: 236 if bad_parms:
216 usage('Got bad arguments %s' % bad_parms) 237 usage('Got bad arguments %s' % bad_parms)
217 238
218 recipe = argv[1] 239 recipe = argv[1]
219 props = argv[2:] 240 props = argv[2:]
220 return dryrun, recipe, props 241 return optparse.Values({'dryrun':dryrun, 'nohooks':nohooks }), recipe, props
221 242
222 243
223 def run_recipe_fetch(recipe, props, aliased=False): 244 def run_recipe_fetch(recipe, props, aliased=False):
224 """Invoke a recipe's fetch method with the passed-through args 245 """Invoke a recipe's fetch method with the passed-through args
225 and return its json output as a python object.""" 246 and return its json output as a python object."""
226 recipe_path = os.path.abspath(os.path.join(SCRIPT_PATH, 'recipes', recipe)) 247 recipe_path = os.path.abspath(os.path.join(SCRIPT_PATH, 'recipes', recipe))
227 if not os.path.exists(recipe_path + '.py'): 248 if not os.path.exists(recipe_path + '.py'):
228 print "Could not find a recipe for %s" % recipe 249 print "Could not find a recipe for %s" % recipe
229 sys.exit(1) 250 sys.exit(1)
230 251
231 cmd = [sys.executable, recipe_path + '.py', 'fetch'] + props 252 cmd = [sys.executable, recipe_path + '.py', 'fetch'] + props
232 result = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0] 253 result = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
233 254
234 spec = json.loads(result) 255 spec = json.loads(result)
235 if 'alias' in spec: 256 if 'alias' in spec:
236 assert not aliased 257 assert not aliased
237 return run_recipe_fetch( 258 return run_recipe_fetch(
238 spec['alias']['recipe'], spec['alias']['props'] + props, aliased=True) 259 spec['alias']['recipe'], spec['alias']['props'] + props, aliased=True)
239 cmd = [sys.executable, recipe_path + '.py', 'root'] 260 cmd = [sys.executable, recipe_path + '.py', 'root']
240 result = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0] 261 result = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
241 root = json.loads(result) 262 root = json.loads(result)
242 return spec, root 263 return spec, root
243 264
244 265
245 def run(dryrun, spec, root): 266 def run(options, spec, root):
246 """Perform a checkout with the given type and configuration. 267 """Perform a checkout with the given type and configuration.
247 268
248 Args: 269 Args:
249 dryrun: if True, don't actually execute the commands 270 options: Options instance.
250 spec: Checkout configuration returned by the the recipe's fetch_spec 271 spec: Checkout configuration returned by the the recipe's fetch_spec
251 method (checkout type, repository url, etc.). 272 method (checkout type, repository url, etc.).
252 root: The directory into which the repo expects to be checkout out. 273 root: The directory into which the repo expects to be checkout out.
253 """ 274 """
254 assert 'type' in spec 275 assert 'type' in spec
255 checkout_type = spec['type'] 276 checkout_type = spec['type']
256 checkout_spec = spec['%s_spec' % checkout_type] 277 checkout_spec = spec['%s_spec' % checkout_type]
257 try: 278 try:
258 checkout = CheckoutFactory(checkout_type, dryrun, checkout_spec, root) 279 checkout = CheckoutFactory(checkout_type, options, checkout_spec, root)
259 except KeyError: 280 except KeyError:
260 return 1 281 return 1
261 if checkout.exists(): 282 if checkout.exists():
262 print 'You appear to already have a checkout. "fetch" is used only' 283 print 'You appear to already have a checkout. "fetch" is used only'
263 print 'to get new checkouts. Use "gclient sync" to update the checkout.' 284 print 'to get new checkouts. Use "gclient sync" to update the checkout.'
264 print 285 print
265 print 'Fetch also does not yet deal with partial checkouts, so if fetch' 286 print 'Fetch also does not yet deal with partial checkouts, so if fetch'
266 print 'failed, delete the checkout and start over (crbug.com/230691).' 287 print 'failed, delete the checkout and start over (crbug.com/230691).'
267 return 1 288 return 1
268 return checkout.init() 289 return checkout.init()
269 290
270 291
271 def main(): 292 def main():
272 dryrun, recipe, props = handle_args(sys.argv) 293 options, recipe, props = handle_args(sys.argv)
273 spec, root = run_recipe_fetch(recipe, props) 294 spec, root = run_recipe_fetch(recipe, props)
274 return run(dryrun, spec, root) 295 return run(options, spec, root)
275 296
276 297
277 if __name__ == '__main__': 298 if __name__ == '__main__':
278 sys.exit(main()) 299 sys.exit(main())
OLDNEW
« no previous file with comments | « no previous file | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698