| OLD | NEW |
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
| 5 | 5 |
| 6 """ | 6 """ |
| 7 Tool to perform checkouts in one easy command line! | 7 Tool to perform checkouts in one easy command line! |
| 8 | 8 |
| 9 Usage: | 9 Usage: |
| 10 fetch <recipe> [--property=value [--property2=value2 ...]] | 10 fetch <recipe> [--property=value [--property2=value2 ...]] |
| 11 | 11 |
| 12 This script is a wrapper around various version control and repository | 12 This script is a wrapper around various version control and repository |
| 13 checkout commands. It requires a |recipe| name, fetches data from that | 13 checkout commands. It requires a |recipe| name, fetches data from that |
| 14 recipe in depot_tools/recipes, and then performs all necessary inits, | 14 recipe in depot_tools/recipes, and then performs all necessary inits, |
| 15 checkouts, pulls, fetches, etc. | 15 checkouts, pulls, fetches, etc. |
| 16 | 16 |
| 17 Optional arguments may be passed on the command line in key-value pairs. | 17 Optional arguments may be passed on the command line in key-value pairs. |
| 18 These parameters will be passed through to the recipe's main method. | 18 These parameters will be passed through to the recipe's main method. |
| 19 """ | 19 """ |
| 20 | 20 |
| 21 import json | 21 import json |
| 22 import optparse |
| 22 import os | 23 import os |
| 23 import subprocess | 24 import subprocess |
| 24 import sys | 25 import sys |
| 25 import pipes | 26 import pipes |
| 26 | 27 |
| 27 from distutils import spawn | 28 from distutils import spawn |
| 28 | 29 |
| 29 | 30 |
| 30 SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__)) | 31 SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__)) |
| 31 | 32 |
| 32 | |
| 33 ################################################# | 33 ################################################# |
| 34 # Checkout class definitions. | 34 # Checkout class definitions. |
| 35 ################################################# | 35 ################################################# |
| 36 class Checkout(object): | 36 class Checkout(object): |
| 37 """Base class for implementing different types of checkouts. | 37 """Base class for implementing different types of checkouts. |
| 38 | 38 |
| 39 Attributes: | 39 Attributes: |
| 40 |base|: the absolute path of the directory in which this script is run. | 40 |base|: the absolute path of the directory in which this script is run. |
| 41 |spec|: the spec for this checkout as returned by the recipe. Different | 41 |spec|: the spec for this checkout as returned by the recipe. Different |
| 42 subclasses will expect different keys in this dictionary. | 42 subclasses will expect different keys in this dictionary. |
| 43 |root|: the directory into which the checkout will be performed, as returned | 43 |root|: the directory into which the checkout will be performed, as returned |
| 44 by the recipe. This is a relative path from |base|. | 44 by the recipe. This is a relative path from |base|. |
| 45 """ | 45 """ |
| 46 def __init__(self, dryrun, spec, root): | 46 def __init__(self, options, spec, root): |
| 47 self.base = os.getcwd() | 47 self.base = os.getcwd() |
| 48 self.dryrun = dryrun | 48 self.options = options |
| 49 self.spec = spec | 49 self.spec = spec |
| 50 self.root = root | 50 self.root = root |
| 51 | 51 |
| 52 def exists(self): | 52 def exists(self): |
| 53 pass | 53 pass |
| 54 | 54 |
| 55 def init(self): | 55 def init(self): |
| 56 pass | 56 pass |
| 57 | 57 |
| 58 def sync(self): | 58 def sync(self): |
| 59 pass | 59 pass |
| 60 | 60 |
| 61 def run(self, cmd, **kwargs): | 61 def run(self, cmd, **kwargs): |
| 62 print 'Running: %s' % (' '.join(pipes.quote(x) for x in cmd)) | 62 print 'Running: %s' % (' '.join(pipes.quote(x) for x in cmd)) |
| 63 if self.dryrun: | 63 if self.options.dryrun: |
| 64 return 0 | 64 return 0 |
| 65 return subprocess.check_call(cmd, **kwargs) | 65 return subprocess.check_call(cmd, **kwargs) |
| 66 | 66 |
| 67 | 67 |
| 68 class GclientCheckout(Checkout): | 68 class GclientCheckout(Checkout): |
| 69 | 69 |
| 70 def run_gclient(self, *cmd, **kwargs): | 70 def run_gclient(self, *cmd, **kwargs): |
| 71 if not spawn.find_executable('gclient'): | 71 if not spawn.find_executable('gclient'): |
| 72 cmd_prefix = (sys.executable, os.path.join(SCRIPT_PATH, 'gclient.py')) | 72 cmd_prefix = (sys.executable, os.path.join(SCRIPT_PATH, 'gclient.py')) |
| 73 else: | 73 else: |
| (...skipping 16 matching lines...) Expand all Loading... |
| 90 def run_svn(self, *cmd, **kwargs): | 90 def run_svn(self, *cmd, **kwargs): |
| 91 if sys.platform == 'win32' and not spawn.find_executable('svn'): | 91 if sys.platform == 'win32' and not spawn.find_executable('svn'): |
| 92 svn_path = os.path.join(SCRIPT_PATH, 'svn_bin', 'svn.exe') | 92 svn_path = os.path.join(SCRIPT_PATH, 'svn_bin', 'svn.exe') |
| 93 else: | 93 else: |
| 94 svn_path = 'svn' | 94 svn_path = 'svn' |
| 95 return self.run((svn_path,) + cmd, **kwargs) | 95 return self.run((svn_path,) + cmd, **kwargs) |
| 96 | 96 |
| 97 | 97 |
| 98 class GclientGitCheckout(GclientCheckout, GitCheckout): | 98 class GclientGitCheckout(GclientCheckout, GitCheckout): |
| 99 | 99 |
| 100 def __init__(self, dryrun, spec, root): | 100 def __init__(self, options, spec, root): |
| 101 super(GclientGitCheckout, self).__init__(dryrun, spec, root) | 101 super(GclientGitCheckout, self).__init__(options, spec, root) |
| 102 assert 'solutions' in self.spec | 102 assert 'solutions' in self.spec |
| 103 keys = ['solutions', 'target_os', 'target_os_only'] | 103 keys = ['solutions', 'target_os', 'target_os_only'] |
| 104 gclient_spec = '\n'.join('%s = %s' % (key, self.spec[key]) | 104 gclient_spec = '\n'.join('%s = %s' % (key, self.spec[key]) |
| 105 for key in keys if key in self.spec) | 105 for key in keys if key in self.spec) |
| 106 self.spec['gclient_spec'] = gclient_spec | 106 self.spec['gclient_spec'] = gclient_spec |
| 107 | 107 |
| 108 def exists(self): | 108 def exists(self): |
| 109 return os.path.exists(os.path.join(os.getcwd(), self.root)) | 109 return os.path.exists(os.path.join(os.getcwd(), self.root)) |
| 110 | 110 |
| 111 def init(self): | 111 def init(self): |
| 112 # TODO(dpranke): Work around issues w/ delta compression on big repos. | 112 # TODO(dpranke): Work around issues w/ delta compression on big repos. |
| 113 self.run_git('config', '--global', 'core.deltaBaseCacheLimit', '1G') | 113 self.run_git('config', '--global', 'core.deltaBaseCacheLimit', '1G') |
| 114 | 114 |
| 115 # Configure and do the gclient checkout. | 115 # Configure and do the gclient checkout. |
| 116 self.run_gclient('config', '--spec', self.spec['gclient_spec']) | 116 self.run_gclient('config', '--spec', self.spec['gclient_spec']) |
| 117 self.run_gclient('sync') | 117 if self.options.nohooks: |
| 118 self.run_gclient('sync', '--nohooks') |
| 119 else: |
| 120 self.run_gclient('sync') |
| 118 | 121 |
| 119 # Configure git. | 122 # Configure git. |
| 120 wd = os.path.join(self.base, self.root) | 123 wd = os.path.join(self.base, self.root) |
| 121 if self.dryrun: | 124 if self.options.dryrun: |
| 122 print 'cd %s' % wd | 125 print 'cd %s' % wd |
| 123 self.run_git( | 126 self.run_git( |
| 124 'submodule', 'foreach', | 127 'submodule', 'foreach', |
| 125 'git config -f $toplevel/.git/config submodule.$name.ignore all', | 128 'git config -f $toplevel/.git/config submodule.$name.ignore all', |
| 126 cwd=wd) | 129 cwd=wd) |
| 127 self.run_git('config', 'diff.ignoreSubmodules', 'all', cwd=wd) | 130 self.run_git('config', 'diff.ignoreSubmodules', 'all', cwd=wd) |
| 128 | 131 |
| 129 | 132 |
| 130 class GclientGitSvnCheckout(GclientGitCheckout, SvnCheckout): | 133 class GclientGitSvnCheckout(GclientGitCheckout, SvnCheckout): |
| 131 | 134 |
| 132 def __init__(self, dryrun, spec, root): | 135 def __init__(self, options, spec, root): |
| 133 super(GclientGitSvnCheckout, self).__init__(dryrun, spec, root) | 136 super(GclientGitSvnCheckout, self).__init__(options, spec, root) |
| 134 assert 'svn_url' in self.spec | 137 assert 'svn_url' in self.spec |
| 135 assert 'svn_branch' in self.spec | 138 assert 'svn_branch' in self.spec |
| 136 assert 'svn_ref' in self.spec | 139 assert 'svn_ref' in self.spec |
| 137 | 140 |
| 138 def init(self): | 141 def init(self): |
| 139 # Ensure we are authenticated with subversion for all submodules. | 142 # Ensure we are authenticated with subversion for all submodules. |
| 140 git_svn_dirs = json.loads(self.spec.get('submodule_git_svn_spec', '{}')) | 143 git_svn_dirs = json.loads(self.spec.get('submodule_git_svn_spec', '{}')) |
| 141 git_svn_dirs.update({self.root: self.spec}) | 144 git_svn_dirs.update({self.root: self.spec}) |
| 142 for _, svn_spec in git_svn_dirs.iteritems(): | 145 for _, svn_spec in git_svn_dirs.iteritems(): |
| 143 try: | 146 try: |
| 144 self.run_svn('ls', '--non-interactive', svn_spec['svn_url']) | 147 self.run_svn('ls', '--non-interactive', svn_spec['svn_url']) |
| 145 except subprocess.CalledProcessError: | 148 except subprocess.CalledProcessError: |
| 146 print 'Please run `svn ls %s`' % svn_spec['svn_url'] | 149 print 'Please run `svn ls %s`' % svn_spec['svn_url'] |
| 147 return 1 | 150 return 1 |
| 148 | 151 |
| 149 super(GclientGitSvnCheckout, self).init() | 152 super(GclientGitSvnCheckout, self).init() |
| 150 | 153 |
| 151 # Configure git-svn. | 154 # Configure git-svn. |
| 152 for path, svn_spec in git_svn_dirs.iteritems(): | 155 for path, svn_spec in git_svn_dirs.iteritems(): |
| 153 real_path = os.path.join(*path.split('/')) | 156 real_path = os.path.join(*path.split('/')) |
| 154 if real_path != self.root: | 157 if real_path != self.root: |
| 155 real_path = os.path.join(self.root, real_path) | 158 real_path = os.path.join(self.root, real_path) |
| 156 wd = os.path.join(self.base, real_path) | 159 wd = os.path.join(self.base, real_path) |
| 157 if self.dryrun: | 160 if self.options.dryrun: |
| 158 print 'cd %s' % wd | 161 print 'cd %s' % wd |
| 159 self.run_git('svn', 'init', '--prefix=origin/', '-T', | 162 self.run_git('svn', 'init', '--prefix=origin/', '-T', |
| 160 svn_spec['svn_branch'], svn_spec['svn_url'], cwd=wd) | 163 svn_spec['svn_branch'], svn_spec['svn_url'], cwd=wd) |
| 161 self.run_git('config', '--replace', 'svn-remote.svn.fetch', | 164 self.run_git('config', '--replace', 'svn-remote.svn.fetch', |
| 162 svn_spec['svn_branch'] + ':refs/remotes/origin/' + | 165 svn_spec['svn_branch'] + ':refs/remotes/origin/' + |
| 163 svn_spec['svn_ref'], cwd=wd) | 166 svn_spec['svn_ref'], cwd=wd) |
| 164 self.run_git('svn', 'fetch', cwd=wd) | 167 self.run_git('svn', 'fetch', cwd=wd) |
| 165 | 168 |
| 166 | 169 |
| 167 | 170 |
| 168 CHECKOUT_TYPE_MAP = { | 171 CHECKOUT_TYPE_MAP = { |
| 169 'gclient': GclientCheckout, | 172 'gclient': GclientCheckout, |
| 170 'gclient_git': GclientGitCheckout, | 173 'gclient_git': GclientGitCheckout, |
| 171 'gclient_git_svn': GclientGitSvnCheckout, | 174 'gclient_git_svn': GclientGitSvnCheckout, |
| 172 'git': GitCheckout, | 175 'git': GitCheckout, |
| 173 } | 176 } |
| 174 | 177 |
| 175 | 178 |
| 176 def CheckoutFactory(type_name, dryrun, spec, root): | 179 def CheckoutFactory(type_name, options, spec, root): |
| 177 """Factory to build Checkout class instances.""" | 180 """Factory to build Checkout class instances.""" |
| 178 class_ = CHECKOUT_TYPE_MAP.get(type_name) | 181 class_ = CHECKOUT_TYPE_MAP.get(type_name) |
| 179 if not class_: | 182 if not class_: |
| 180 raise KeyError('unrecognized checkout type: %s' % type_name) | 183 raise KeyError('unrecognized checkout type: %s' % type_name) |
| 181 return class_(dryrun, spec, root) | 184 return class_(options, spec, root) |
| 182 | 185 |
| 183 | 186 |
| 184 ################################################# | 187 ################################################# |
| 185 # Utility function and file entry point. | 188 # Utility function and file entry point. |
| 186 ################################################# | 189 ################################################# |
| 187 def usage(msg=None): | 190 def usage(msg=None): |
| 188 """Print help and exit.""" | 191 """Print help and exit.""" |
| 189 if msg: | 192 if msg: |
| 190 print 'Error:', msg | 193 print 'Error:', msg |
| 191 | 194 |
| 192 print ( | 195 print ( |
| 193 """ | 196 """ |
| 194 usage: %s [-n|--dry-run] <recipe> [--property=value [--property2=value2 ...]] | 197 usage: %s [options] <recipe> [--property=value [--property2=value2 ...]] |
| 198 |
| 199 This script can be used to download the Chromium sources. See |
| 200 http://www.chromium.org/developers/how-tos/get-the-code |
| 201 for full usage instructions. |
| 202 |
| 203 Valid options: |
| 204 -h, --help, help Print this message. |
| 205 --nohooks Don't run hooks after checkout. |
| 206 -n, --dryrun Don't run commands, only print them. |
| 195 """ % os.path.basename(sys.argv[0])) | 207 """ % os.path.basename(sys.argv[0])) |
| 196 sys.exit(bool(msg)) | 208 sys.exit(bool(msg)) |
| 197 | 209 |
| 198 | 210 |
| 199 def handle_args(argv): | 211 def handle_args(argv): |
| 200 """Gets the recipe name from the command line arguments.""" | 212 """Gets the recipe name from the command line arguments.""" |
| 201 if len(argv) <= 1: | 213 if len(argv) <= 1: |
| 202 usage('Must specify a recipe.') | 214 usage('Must specify a recipe.') |
| 203 if argv[1] in ('-h', '--help', 'help'): | 215 if argv[1] in ('-h', '--help', 'help'): |
| 204 usage() | 216 usage() |
| 205 | 217 |
| 206 dryrun = False | 218 dryrun = False |
| 207 if argv[1] in ('-n', '--dry-run'): | 219 nohooks = False |
| 208 dryrun = True | 220 while len(argv) >= 2: |
| 221 arg = argv[1] |
| 222 if not arg.startswith('-'): |
| 223 break |
| 209 argv.pop(1) | 224 argv.pop(1) |
| 225 if arg in ('-n', '--dry-run'): |
| 226 dryrun = True |
| 227 elif arg == '--nohooks': |
| 228 nohooks = True |
| 229 else: |
| 230 usage('Invalid option %s.' % arg) |
| 210 | 231 |
| 211 def looks_like_arg(arg): | 232 def looks_like_arg(arg): |
| 212 return arg.startswith('--') and arg.count('=') == 1 | 233 return arg.startswith('--') and arg.count('=') == 1 |
| 213 | 234 |
| 214 bad_parms = [x for x in argv[2:] if not looks_like_arg(x)] | 235 bad_parms = [x for x in argv[2:] if not looks_like_arg(x)] |
| 215 if bad_parms: | 236 if bad_parms: |
| 216 usage('Got bad arguments %s' % bad_parms) | 237 usage('Got bad arguments %s' % bad_parms) |
| 217 | 238 |
| 218 recipe = argv[1] | 239 recipe = argv[1] |
| 219 props = argv[2:] | 240 props = argv[2:] |
| 220 return dryrun, recipe, props | 241 return optparse.Values({'dryrun':dryrun, 'nohooks':nohooks }), recipe, props |
| 221 | 242 |
| 222 | 243 |
| 223 def run_recipe_fetch(recipe, props, aliased=False): | 244 def run_recipe_fetch(recipe, props, aliased=False): |
| 224 """Invoke a recipe's fetch method with the passed-through args | 245 """Invoke a recipe's fetch method with the passed-through args |
| 225 and return its json output as a python object.""" | 246 and return its json output as a python object.""" |
| 226 recipe_path = os.path.abspath(os.path.join(SCRIPT_PATH, 'recipes', recipe)) | 247 recipe_path = os.path.abspath(os.path.join(SCRIPT_PATH, 'recipes', recipe)) |
| 227 if not os.path.exists(recipe_path + '.py'): | 248 if not os.path.exists(recipe_path + '.py'): |
| 228 print "Could not find a recipe for %s" % recipe | 249 print "Could not find a recipe for %s" % recipe |
| 229 sys.exit(1) | 250 sys.exit(1) |
| 230 | 251 |
| 231 cmd = [sys.executable, recipe_path + '.py', 'fetch'] + props | 252 cmd = [sys.executable, recipe_path + '.py', 'fetch'] + props |
| 232 result = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0] | 253 result = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0] |
| 233 | 254 |
| 234 spec = json.loads(result) | 255 spec = json.loads(result) |
| 235 if 'alias' in spec: | 256 if 'alias' in spec: |
| 236 assert not aliased | 257 assert not aliased |
| 237 return run_recipe_fetch( | 258 return run_recipe_fetch( |
| 238 spec['alias']['recipe'], spec['alias']['props'] + props, aliased=True) | 259 spec['alias']['recipe'], spec['alias']['props'] + props, aliased=True) |
| 239 cmd = [sys.executable, recipe_path + '.py', 'root'] | 260 cmd = [sys.executable, recipe_path + '.py', 'root'] |
| 240 result = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0] | 261 result = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0] |
| 241 root = json.loads(result) | 262 root = json.loads(result) |
| 242 return spec, root | 263 return spec, root |
| 243 | 264 |
| 244 | 265 |
| 245 def run(dryrun, spec, root): | 266 def run(options, spec, root): |
| 246 """Perform a checkout with the given type and configuration. | 267 """Perform a checkout with the given type and configuration. |
| 247 | 268 |
| 248 Args: | 269 Args: |
| 249 dryrun: if True, don't actually execute the commands | 270 options: Options instance. |
| 250 spec: Checkout configuration returned by the the recipe's fetch_spec | 271 spec: Checkout configuration returned by the the recipe's fetch_spec |
| 251 method (checkout type, repository url, etc.). | 272 method (checkout type, repository url, etc.). |
| 252 root: The directory into which the repo expects to be checkout out. | 273 root: The directory into which the repo expects to be checkout out. |
| 253 """ | 274 """ |
| 254 assert 'type' in spec | 275 assert 'type' in spec |
| 255 checkout_type = spec['type'] | 276 checkout_type = spec['type'] |
| 256 checkout_spec = spec['%s_spec' % checkout_type] | 277 checkout_spec = spec['%s_spec' % checkout_type] |
| 257 try: | 278 try: |
| 258 checkout = CheckoutFactory(checkout_type, dryrun, checkout_spec, root) | 279 checkout = CheckoutFactory(checkout_type, options, checkout_spec, root) |
| 259 except KeyError: | 280 except KeyError: |
| 260 return 1 | 281 return 1 |
| 261 if checkout.exists(): | 282 if checkout.exists(): |
| 262 print 'You appear to already have a checkout. "fetch" is used only' | 283 print 'You appear to already have a checkout. "fetch" is used only' |
| 263 print 'to get new checkouts. Use "gclient sync" to update the checkout.' | 284 print 'to get new checkouts. Use "gclient sync" to update the checkout.' |
| 264 print | 285 print |
| 265 print 'Fetch also does not yet deal with partial checkouts, so if fetch' | 286 print 'Fetch also does not yet deal with partial checkouts, so if fetch' |
| 266 print 'failed, delete the checkout and start over (crbug.com/230691).' | 287 print 'failed, delete the checkout and start over (crbug.com/230691).' |
| 267 return 1 | 288 return 1 |
| 268 return checkout.init() | 289 return checkout.init() |
| 269 | 290 |
| 270 | 291 |
| 271 def main(): | 292 def main(): |
| 272 dryrun, recipe, props = handle_args(sys.argv) | 293 options, recipe, props = handle_args(sys.argv) |
| 273 spec, root = run_recipe_fetch(recipe, props) | 294 spec, root = run_recipe_fetch(recipe, props) |
| 274 return run(dryrun, spec, root) | 295 return run(options, spec, root) |
| 275 | 296 |
| 276 | 297 |
| 277 if __name__ == '__main__': | 298 if __name__ == '__main__': |
| 278 sys.exit(main()) | 299 sys.exit(main()) |
| OLD | NEW |