| OLD | NEW |
| 1 #!/usr/bin/python | 1 #!/usr/bin/python |
| 2 # Copyright (c) 2010 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2010 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
| 5 | 5 |
| 6 """Meta checkout manager supporting both Subversion and GIT. | 6 """Meta checkout manager supporting both Subversion and GIT. |
| 7 | 7 |
| 8 Files | 8 Files |
| 9 .gclient : Current client configuration, written by 'config' command. | 9 .gclient : Current client configuration, written by 'config' command. |
| 10 Format is a Python script defining 'solutions', a list whose | 10 Format is a Python script defining 'solutions', a list whose |
| (...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 42 it will be removed from the list and the list will be extended | 42 it will be removed from the list and the list will be extended |
| 43 by the list of matching files. | 43 by the list of matching files. |
| 44 | 44 |
| 45 Example: | 45 Example: |
| 46 hooks = [ | 46 hooks = [ |
| 47 { "pattern": "\\.(gif|jpe?g|pr0n|png)$", | 47 { "pattern": "\\.(gif|jpe?g|pr0n|png)$", |
| 48 "action": ["python", "image_indexer.py", "--all"]}, | 48 "action": ["python", "image_indexer.py", "--all"]}, |
| 49 ] | 49 ] |
| 50 """ | 50 """ |
| 51 | 51 |
| 52 __version__ = "0.4.1" | 52 __version__ = "0.5" |
| 53 | 53 |
| 54 import errno | |
| 55 import logging | 54 import logging |
| 56 import optparse | 55 import optparse |
| 57 import os | 56 import os |
| 58 import pprint | 57 import pprint |
| 59 import re | 58 import re |
| 60 import subprocess | 59 import subprocess |
| 61 import sys | 60 import sys |
| 62 import urlparse | 61 import urlparse |
| 63 import urllib | 62 import urllib |
| 64 | 63 |
| (...skipping 25 matching lines...) Expand all Loading... |
| 90 | 89 |
| 91 sub_target_name is an optional parameter if the module name in the other | 90 sub_target_name is an optional parameter if the module name in the other |
| 92 DEPS file is different. E.g., you might want to map src/net to net.""" | 91 DEPS file is different. E.g., you might want to map src/net to net.""" |
| 93 self.module_name = module_name | 92 self.module_name = module_name |
| 94 self.sub_target_name = sub_target_name | 93 self.sub_target_name = sub_target_name |
| 95 | 94 |
| 96 def __str__(self): | 95 def __str__(self): |
| 97 return 'From(%s, %s)' % (repr(self.module_name), | 96 return 'From(%s, %s)' % (repr(self.module_name), |
| 98 repr(self.sub_target_name)) | 97 repr(self.sub_target_name)) |
| 99 | 98 |
| 100 def GetUrl(self, target_name, sub_deps_base_url, root_dir, sub_deps): | |
| 101 """Resolve the URL for this From entry.""" | |
| 102 sub_deps_target_name = target_name | |
| 103 if self.sub_target_name: | |
| 104 sub_deps_target_name = self.sub_target_name | |
| 105 url = sub_deps[sub_deps_target_name] | |
| 106 if url.startswith('/'): | |
| 107 # If it's a relative URL, we need to resolve the URL relative to the | |
| 108 # sub deps base URL. | |
| 109 if not isinstance(sub_deps_base_url, basestring): | |
| 110 sub_deps_base_url = sub_deps_base_url.GetPath() | |
| 111 scm = gclient_scm.CreateSCM(sub_deps_base_url, root_dir, | |
| 112 None) | |
| 113 url = scm.FullUrlForRelativeUrl(url) | |
| 114 return url | |
| 115 | |
| 116 class FileImpl(object): | 99 class FileImpl(object): |
| 117 """Used to implement the File('') syntax which lets you sync a single file | 100 """Used to implement the File('') syntax which lets you sync a single file |
| 118 from an SVN repo.""" | 101 from an SVN repo.""" |
| 119 | 102 |
| 120 def __init__(self, file_location): | 103 def __init__(self, file_location): |
| 121 self.file_location = file_location | 104 self.file_location = file_location |
| 122 | 105 |
| 123 def __str__(self): | 106 def __str__(self): |
| 124 return 'File("%s")' % self.file_location | 107 return 'File("%s")' % self.file_location |
| 125 | 108 |
| (...skipping 27 matching lines...) Expand all Loading... |
| 153 class Dependency(GClientKeywords): | 136 class Dependency(GClientKeywords): |
| 154 """Object that represents a dependency checkout.""" | 137 """Object that represents a dependency checkout.""" |
| 155 DEPS_FILE = 'DEPS' | 138 DEPS_FILE = 'DEPS' |
| 156 | 139 |
| 157 def __init__(self, parent, name, url, safesync_url=None, custom_deps=None, | 140 def __init__(self, parent, name, url, safesync_url=None, custom_deps=None, |
| 158 custom_vars=None, deps_file=None): | 141 custom_vars=None, deps_file=None): |
| 159 GClientKeywords.__init__(self) | 142 GClientKeywords.__init__(self) |
| 160 self.parent = parent | 143 self.parent = parent |
| 161 self.name = name | 144 self.name = name |
| 162 self.url = url | 145 self.url = url |
| 146 self.parsed_url = None |
| 163 # These 2 are only set in .gclient and not in DEPS files. | 147 # These 2 are only set in .gclient and not in DEPS files. |
| 164 self.safesync_url = safesync_url | 148 self.safesync_url = safesync_url |
| 165 self.custom_vars = custom_vars or {} | 149 self.custom_vars = custom_vars or {} |
| 166 self.custom_deps = custom_deps or {} | 150 self.custom_deps = custom_deps or {} |
| 167 self.deps_hooks = [] | 151 self.deps_hooks = [] |
| 168 self.dependencies = [] | 152 self.dependencies = [] |
| 169 self.deps_file = deps_file or self.DEPS_FILE | 153 self.deps_file = deps_file or self.DEPS_FILE |
| 154 # A cache of the files affected by the current operation, necessary for |
| 155 # hooks. |
| 156 self.file_list = [] |
| 170 self.deps_parsed = False | 157 self.deps_parsed = False |
| 171 self.direct_reference = False | 158 self.direct_reference = False |
| 172 | 159 |
| 173 # Sanity checks | 160 # Sanity checks |
| 174 if not self.name and self.parent: | 161 if not self.name and self.parent: |
| 175 raise gclient_utils.Error('Dependency without name') | 162 raise gclient_utils.Error('Dependency without name') |
| 163 if self.name in [d.name for d in self.tree(False)]: |
| 164 raise gclient_utils.Error('Dependency %s specified more than once' % |
| 165 self.name) |
| 176 if not isinstance(self.url, | 166 if not isinstance(self.url, |
| 177 (basestring, self.FromImpl, self.FileImpl, None.__class__)): | 167 (basestring, self.FromImpl, self.FileImpl, None.__class__)): |
| 178 raise gclient_utils.Error('dependency url must be either a string, None, ' | 168 raise gclient_utils.Error('dependency url must be either a string, None, ' |
| 179 'File() or From() instead of %s' % | 169 'File() or From() instead of %s' % |
| 180 self.url.__class__.__name__) | 170 self.url.__class__.__name__) |
| 181 if '/' in self.deps_file or '\\' in self.deps_file: | 171 if '/' in self.deps_file or '\\' in self.deps_file: |
| 182 raise gclient_utils.Error('deps_file name must not be a path, just a ' | 172 raise gclient_utils.Error('deps_file name must not be a path, just a ' |
| 183 'filename. %s' % self.deps_file) | 173 'filename. %s' % self.deps_file) |
| 184 | 174 |
| 175 def LateOverride(self, url): |
| 176 overriden_url = self.get_custom_deps(self.name, url) |
| 177 if overriden_url != url: |
| 178 self.parsed_url = overriden_url |
| 179 logging.debug('%s, %s was overriden to %s' % (self.name, url, |
| 180 self.parsed_url)) |
| 181 elif isinstance(url, self.FromImpl): |
| 182 ref = [dep for dep in self.tree(True) if url.module_name == dep.name] |
| 183 if not len(ref) == 1: |
| 184 raise Exception('Failed to find one reference to %s. %s' % ( |
| 185 url.module_name, ref)) |
| 186 ref = ref[0] |
| 187 sub_target = url.sub_target_name or url |
| 188 # Make sure the referenced dependency DEPS file is loaded and file the |
| 189 # inner referenced dependency. |
| 190 ref.ParseDepsFile(False) |
| 191 found_dep = None |
| 192 for d in ref.dependencies: |
| 193 if d.name == sub_target: |
| 194 found_dep = d |
| 195 break |
| 196 if not found_dep: |
| 197 raise Exception('Couldn\'t find %s in %s, referenced by %s' % ( |
| 198 sub_target, ref.name, self.name)) |
| 199 # Call LateOverride() again. |
| 200 self.parsed_url = found_dep.LateOverride(found_dep.url) |
| 201 logging.debug('%s, %s to %s' % (self.name, url, self.parsed_url)) |
| 202 elif isinstance(url, basestring): |
| 203 parsed_url = urlparse.urlparse(url) |
| 204 if not parsed_url[0]: |
| 205 # A relative url. Fetch the real base. |
| 206 path = parsed_url[2] |
| 207 if not path.startswith('/'): |
| 208 raise gclient_utils.Error( |
| 209 'relative DEPS entry \'%s\' must begin with a slash' % url) |
| 210 # Create a scm just to query the full url. |
| 211 scm = gclient_scm.CreateSCM(self.parent.parsed_url, self.root_dir(), |
| 212 None) |
| 213 self.parsed_url = scm.FullUrlForRelativeUrl(url) |
| 214 else: |
| 215 self.parsed_url = url |
| 216 logging.debug('%s, %s -> %s' % (self.name, url, self.parsed_url)) |
| 217 elif isinstance(url, self.FileImpl): |
| 218 self.parsed_url = url |
| 219 logging.debug('%s, %s -> %s (File)' % (self.name, url, self.parsed_url)) |
| 220 return self.parsed_url |
| 221 |
| 185 def ParseDepsFile(self, direct_reference): | 222 def ParseDepsFile(self, direct_reference): |
| 186 """Parses the DEPS file for this dependency.""" | 223 """Parses the DEPS file for this dependency.""" |
| 187 if direct_reference: | 224 if direct_reference: |
| 188 # Maybe it was referenced earlier by a From() keyword but it's now | 225 # Maybe it was referenced earlier by a From() keyword but it's now |
| 189 # directly referenced. | 226 # directly referenced. |
| 190 self.direct_reference = direct_reference | 227 self.direct_reference = direct_reference |
| 228 if self.deps_parsed: |
| 229 return |
| 191 self.deps_parsed = True | 230 self.deps_parsed = True |
| 192 filepath = os.path.join(self.root_dir(), self.name, self.deps_file) | 231 filepath = os.path.join(self.root_dir(), self.name, self.deps_file) |
| 193 if not os.path.isfile(filepath): | 232 if not os.path.isfile(filepath): |
| 194 return {} | 233 return |
| 195 deps_content = gclient_utils.FileRead(filepath) | 234 deps_content = gclient_utils.FileRead(filepath) |
| 196 | 235 |
| 197 # Eval the content. | 236 # Eval the content. |
| 198 # One thing is unintuitive, vars= {} must happen before Var() use. | 237 # One thing is unintuitive, vars= {} must happen before Var() use. |
| 199 local_scope = {} | 238 local_scope = {} |
| 200 var = self.VarImpl(self.custom_vars, local_scope) | 239 var = self.VarImpl(self.custom_vars, local_scope) |
| 201 global_scope = { | 240 global_scope = { |
| 202 'File': self.FileImpl, | 241 'File': self.FileImpl, |
| 203 'From': self.FromImpl, | 242 'From': self.FromImpl, |
| 204 'Var': var.Lookup, | 243 'Var': var.Lookup, |
| (...skipping 30 matching lines...) Expand all Loading... |
| 235 # the dictionary using paths relative to the directory containing | 274 # the dictionary using paths relative to the directory containing |
| 236 # the DEPS file. | 275 # the DEPS file. |
| 237 use_relative_paths = local_scope.get('use_relative_paths', False) | 276 use_relative_paths = local_scope.get('use_relative_paths', False) |
| 238 if use_relative_paths: | 277 if use_relative_paths: |
| 239 rel_deps = {} | 278 rel_deps = {} |
| 240 for d, url in deps.items(): | 279 for d, url in deps.items(): |
| 241 # normpath is required to allow DEPS to use .. in their | 280 # normpath is required to allow DEPS to use .. in their |
| 242 # dependency local path. | 281 # dependency local path. |
| 243 rel_deps[os.path.normpath(os.path.join(self.name, d))] = url | 282 rel_deps[os.path.normpath(os.path.join(self.name, d))] = url |
| 244 deps = rel_deps | 283 deps = rel_deps |
| 245 # TODO(maruel): Add these dependencies into self.dependencies. | |
| 246 return deps | |
| 247 | 284 |
| 248 def _ParseAllDeps(self, solution_urls): | 285 # Convert the deps into real Dependency. |
| 249 """Parse the complete list of dependencies for the client. | 286 for name, url in deps.iteritems(): |
| 287 if name in [s.name for s in self.dependencies]: |
| 288 raise |
| 289 self.dependencies.append(Dependency(self, name, url)) |
| 290 # Sort by name. |
| 291 self.dependencies.sort(key=lambda x: x.name) |
| 292 logging.info('Loaded: %s' % str(self)) |
| 250 | 293 |
| 251 Args: | 294 def RunCommandRecursively(self, options, revision_overrides, |
| 252 solution_urls: A dict mapping module names (as relative paths) to URLs | 295 command, args, pm): |
| 253 corresponding to the solutions specified by the client. This parameter | 296 """Runs 'command' before parsing the DEPS in case it's a initial checkout |
| 254 is passed as an optimization. | 297 or a revert.""" |
| 298 assert self.file_list == [] |
| 299 # When running runhooks, there's no need to consult the SCM. |
| 300 # All known hooks are expected to run unconditionally regardless of working |
| 301 # copy state, so skip the SCM status check. |
| 302 run_scm = command not in ('runhooks', None) |
| 303 self.LateOverride(self.url) |
| 304 if run_scm and self.parsed_url: |
| 305 if isinstance(self.parsed_url, self.FileImpl): |
| 306 # Special support for single-file checkout. |
| 307 options.revision = self.parsed_url.GetRevision() |
| 308 scm = gclient_scm.CreateSCM(self.parsed_url.GetPath(), self.root_dir(), |
| 309 self.name) |
| 310 scm.RunCommand("updatesingle", options, |
| 311 args + [self.parsed_url.GetFilename()], self.file_list) |
| 312 else: |
| 313 options.revision = revision_overrides.get(self.name) |
| 314 scm = gclient_scm.CreateSCM(self.parsed_url, self.root_dir(), self.name) |
| 315 scm.RunCommand(command, options, args, self.file_list) |
| 316 self.file_list = [os.path.join(self.name, f.strip()) |
| 317 for f in self.file_list] |
| 318 options.revision = None |
| 319 if pm: |
| 320 # The + 1 comes from the fact that .gclient is considered a step in |
| 321 # itself, .i.e. this code is called one time for the .gclient. This is not |
| 322 # conceptually correct but it simplifies code. |
| 323 pm._total = len(self.tree(False)) + 1 |
| 324 pm.update() |
| 325 if self.recursion_limit(): |
| 326 # Then we can parse the DEPS file. |
| 327 self.ParseDepsFile(True) |
| 328 if pm: |
| 329 pm._total = len(self.tree(False)) + 1 |
| 330 pm.update(0) |
| 331 # Parse the dependencies of this dependency. |
| 332 for s in self.dependencies: |
| 333 # TODO(maruel): All these can run concurrently! No need for threads, |
| 334 # just buffer stdout&stderr on pipes and flush as they complete. |
| 335 # Watch out for stdin. |
| 336 s.RunCommandRecursively(options, revision_overrides, command, args, pm) |
| 255 | 337 |
| 256 Returns: | 338 def RunHooksRecursively(self, options): |
| 257 A dict mapping module names (as relative paths) to URLs corresponding | 339 """Evaluates all hooks, running actions as needed. RunCommandRecursively() |
| 258 to the entire set of dependencies to checkout for the given client. | 340 must have been called before to load the DEPS.""" |
| 341 # If "--force" was specified, run all hooks regardless of what files have |
| 342 # changed. |
| 343 if self.deps_hooks: |
| 344 # TODO(maruel): If the user is using git or git-svn, then we don't know |
| 345 # what files have changed so we always run all hooks. It'd be nice to fix |
| 346 # that. |
| 347 if (options.force or |
| 348 gclient_scm.GetScmName(self.parsed_url) in ('git', None) or |
| 349 os.path.isdir(os.path.join(self.root_dir(), self.name, '.git'))): |
| 350 for hook_dict in self.deps_hooks: |
| 351 self._RunHookAction(hook_dict, []) |
| 352 else: |
| 353 # TODO(phajdan.jr): We should know exactly when the paths are absolute. |
| 354 # Convert all absolute paths to relative. |
| 355 for i in range(len(self.file_list)): |
| 356 # It depends on the command being executed (like runhooks vs sync). |
| 357 if not os.path.isabs(self.file_list[i]): |
| 358 continue |
| 259 | 359 |
| 260 Raises: | 360 prefix = os.path.commonprefix([self.root_dir().lower(), |
| 261 Error: If a dependency conflicts with another dependency or of a solution. | 361 self.file_list[i].lower()]) |
| 262 """ | 362 self.file_list[i] = self.file_list[i][len(prefix):] |
| 263 deps = {} | |
| 264 for solution in self.dependencies: | |
| 265 solution_deps = solution.ParseDepsFile(True) | |
| 266 | 363 |
| 267 # If a line is in custom_deps, but not in the solution, we want to append | 364 # Strip any leading path separators. |
| 268 # this line to the solution. | 365 while file_list[i].startswith('\\') or file_list[i].startswith('/'): |
| 269 for d in solution.custom_deps: | 366 self.file_list[i] = self.file_list[i][1:] |
| 270 if d not in solution_deps: | |
| 271 solution_deps[d] = solution.custom_deps[d] | |
| 272 | 367 |
| 273 for d in solution_deps: | 368 # Run hooks on the basis of whether the files from the gclient operation |
| 274 if d in solution.custom_deps: | 369 # match each hook's pattern. |
| 275 # Dependency is overriden. | 370 for hook_dict in self.deps_hooks: |
| 276 url = solution.custom_deps[d] | 371 pattern = re.compile(hook_dict['pattern']) |
| 277 if url is None: | 372 matching_file_list = [f for f in self.file_list if pattern.search(f)] |
| 278 continue | 373 if matching_file_list: |
| 279 else: | 374 self._RunHookAction(hook_dict, matching_file_list) |
| 280 url = solution_deps[d] | 375 if self.recursion_limit(): |
| 281 # if we have a From reference dependent on another solution, then | 376 for s in self.dependencies: |
| 282 # just skip the From reference. When we pull deps for the solution, | 377 s.RunHooksRecursively(options) |
| 283 # we will take care of this dependency. | |
| 284 # | |
| 285 # If multiple solutions all have the same From reference, then we | |
| 286 # should only add one to our list of dependencies. | |
| 287 if isinstance(url, self.FromImpl): | |
| 288 if url.module_name in solution_urls: | |
| 289 # Already parsed. | |
| 290 continue | |
| 291 if d in deps and type(deps[d]) != str: | |
| 292 if url.module_name == deps[d].module_name: | |
| 293 continue | |
| 294 elif isinstance(url, str): | |
| 295 parsed_url = urlparse.urlparse(url) | |
| 296 scheme = parsed_url[0] | |
| 297 if not scheme: | |
| 298 # A relative url. Fetch the real base. | |
| 299 path = parsed_url[2] | |
| 300 if path[0] != "/": | |
| 301 raise gclient_utils.Error( | |
| 302 "relative DEPS entry \"%s\" must begin with a slash" % d) | |
| 303 # Create a scm just to query the full url. | |
| 304 scm = gclient_scm.CreateSCM(solution.url, self.root_dir(), | |
| 305 None) | |
| 306 url = scm.FullUrlForRelativeUrl(url) | |
| 307 if d in deps and deps[d] != url: | |
| 308 raise gclient_utils.Error( | |
| 309 "Solutions have conflicting versions of dependency \"%s\"" % d) | |
| 310 if d in solution_urls and solution_urls[d] != url: | |
| 311 raise gclient_utils.Error( | |
| 312 "Dependency \"%s\" conflicts with specified solution" % d) | |
| 313 # Grab the dependency. | |
| 314 deps[d] = url | |
| 315 return deps | |
| 316 | |
| 317 def _RunHooks(self, command, file_list, is_using_git): | |
| 318 """Evaluates all hooks, running actions as needed. | |
| 319 """ | |
| 320 # Hooks only run for these command types. | |
| 321 if not command in ('update', 'revert', 'runhooks'): | |
| 322 return | |
| 323 | |
| 324 # Hooks only run when --nohooks is not specified | |
| 325 if self._options.nohooks: | |
| 326 return | |
| 327 | |
| 328 # Get any hooks from the .gclient file. | |
| 329 hooks = self.deps_hooks[:] | |
| 330 # Add any hooks found in DEPS files. | |
| 331 for d in self.dependencies: | |
| 332 hooks.extend(d.deps_hooks) | |
| 333 | |
| 334 # If "--force" was specified, run all hooks regardless of what files have | |
| 335 # changed. If the user is using git, then we don't know what files have | |
| 336 # changed so we always run all hooks. | |
| 337 if self._options.force or is_using_git: | |
| 338 for hook_dict in hooks: | |
| 339 self._RunHookAction(hook_dict, []) | |
| 340 return | |
| 341 | |
| 342 # Run hooks on the basis of whether the files from the gclient operation | |
| 343 # match each hook's pattern. | |
| 344 for hook_dict in hooks: | |
| 345 pattern = re.compile(hook_dict['pattern']) | |
| 346 matching_file_list = [f for f in file_list if pattern.search(f)] | |
| 347 if matching_file_list: | |
| 348 self._RunHookAction(hook_dict, matching_file_list) | |
| 349 | 378 |
| 350 def _RunHookAction(self, hook_dict, matching_file_list): | 379 def _RunHookAction(self, hook_dict, matching_file_list): |
| 351 """Runs the action from a single hook.""" | 380 """Runs the action from a single hook.""" |
| 352 logging.info(hook_dict) | 381 logging.info(hook_dict) |
| 353 logging.info(matching_file_list) | 382 logging.info(matching_file_list) |
| 354 command = hook_dict['action'][:] | 383 command = hook_dict['action'][:] |
| 355 if command[0] == 'python': | 384 if command[0] == 'python': |
| 356 # If the hook specified "python" as the first item, the action is a | 385 # If the hook specified "python" as the first item, the action is a |
| 357 # Python script. Run it by starting a new copy of the same | 386 # Python script. Run it by starting a new copy of the same |
| 358 # interpreter. | 387 # interpreter. |
| (...skipping 23 matching lines...) Expand all Loading... |
| 382 def get_custom_deps(self, name, url): | 411 def get_custom_deps(self, name, url): |
| 383 """Returns a custom deps if applicable.""" | 412 """Returns a custom deps if applicable.""" |
| 384 if self.parent: | 413 if self.parent: |
| 385 url = self.parent.get_custom_deps(name, url) | 414 url = self.parent.get_custom_deps(name, url) |
| 386 # None is a valid return value to disable a dependency. | 415 # None is a valid return value to disable a dependency. |
| 387 return self.custom_deps.get(name, url) | 416 return self.custom_deps.get(name, url) |
| 388 | 417 |
| 389 def __str__(self): | 418 def __str__(self): |
| 390 out = [] | 419 out = [] |
| 391 for i in ('name', 'url', 'safesync_url', 'custom_deps', 'custom_vars', | 420 for i in ('name', 'url', 'safesync_url', 'custom_deps', 'custom_vars', |
| 392 'deps_hooks'): | 421 'deps_hooks', 'file_list'): |
| 393 # 'deps_file' | 422 # 'deps_file' |
| 394 if self.__dict__[i]: | 423 if self.__dict__[i]: |
| 395 out.append('%s: %s' % (i, self.__dict__[i])) | 424 out.append('%s: %s' % (i, self.__dict__[i])) |
| 396 | 425 |
| 397 for d in self.dependencies: | 426 for d in self.dependencies: |
| 398 out.extend([' ' + x for x in str(d).splitlines()]) | 427 out.extend([' ' + x for x in str(d).splitlines()]) |
| 399 out.append('') | 428 out.append('') |
| 400 return '\n'.join(out) | 429 return '\n'.join(out) |
| 401 | 430 |
| 402 def __repr__(self): | 431 def __repr__(self): |
| (...skipping 95 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 498 os.path.join(path, options.config_filename))) | 527 os.path.join(path, options.config_filename))) |
| 499 return client | 528 return client |
| 500 | 529 |
| 501 def SetDefaultConfig(self, solution_name, solution_url, safesync_url): | 530 def SetDefaultConfig(self, solution_name, solution_url, safesync_url): |
| 502 self.SetConfig(self.DEFAULT_CLIENT_FILE_TEXT % { | 531 self.SetConfig(self.DEFAULT_CLIENT_FILE_TEXT % { |
| 503 'solution_name': solution_name, | 532 'solution_name': solution_name, |
| 504 'solution_url': solution_url, | 533 'solution_url': solution_url, |
| 505 'safesync_url' : safesync_url, | 534 'safesync_url' : safesync_url, |
| 506 }) | 535 }) |
| 507 | 536 |
| 508 def _SaveEntries(self, entries): | 537 def _SaveEntries(self): |
| 509 """Creates a .gclient_entries file to record the list of unique checkouts. | 538 """Creates a .gclient_entries file to record the list of unique checkouts. |
| 510 | 539 |
| 511 The .gclient_entries file lives in the same directory as .gclient. | 540 The .gclient_entries file lives in the same directory as .gclient. |
| 512 """ | 541 """ |
| 513 # Sometimes pprint.pformat will use {', sometimes it'll use { ' ... It | 542 # Sometimes pprint.pformat will use {', sometimes it'll use { ' ... It |
| 514 # makes testing a bit too fun. | 543 # makes testing a bit too fun. |
| 515 result = pprint.pformat(entries, 2) | 544 result = 'entries = {\n' |
| 516 if result.startswith('{\''): | 545 for entry in self.tree(False): |
| 517 result = '{ \'' + result[2:] | 546 result += ' %s: %s,\n' % (pprint.pformat(entry.name), |
| 518 text = 'entries = \\\n' + result + '\n' | 547 pprint.pformat(entry.parsed_url)) |
| 548 result += '}\n' |
| 519 file_path = os.path.join(self.root_dir(), self._options.entries_filename) | 549 file_path = os.path.join(self.root_dir(), self._options.entries_filename) |
| 520 gclient_utils.FileWrite(file_path, text) | 550 logging.info(result) |
| 551 gclient_utils.FileWrite(file_path, result) |
| 521 | 552 |
| 522 def _ReadEntries(self): | 553 def _ReadEntries(self): |
| 523 """Read the .gclient_entries file for the given client. | 554 """Read the .gclient_entries file for the given client. |
| 524 | 555 |
| 525 Returns: | 556 Returns: |
| 526 A sequence of solution names, which will be empty if there is the | 557 A sequence of solution names, which will be empty if there is the |
| 527 entries file hasn't been created yet. | 558 entries file hasn't been created yet. |
| 528 """ | 559 """ |
| 529 scope = {} | 560 scope = {} |
| 530 filename = os.path.join(self.root_dir(), self._options.entries_filename) | 561 filename = os.path.join(self.root_dir(), self._options.entries_filename) |
| (...skipping 40 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 571 def RunOnDeps(self, command, args): | 602 def RunOnDeps(self, command, args): |
| 572 """Runs a command on each dependency in a client and its dependencies. | 603 """Runs a command on each dependency in a client and its dependencies. |
| 573 | 604 |
| 574 Args: | 605 Args: |
| 575 command: The command to use (e.g., 'status' or 'diff') | 606 command: The command to use (e.g., 'status' or 'diff') |
| 576 args: list of str - extra arguments to add to the command line. | 607 args: list of str - extra arguments to add to the command line. |
| 577 """ | 608 """ |
| 578 if not self.dependencies: | 609 if not self.dependencies: |
| 579 raise gclient_utils.Error('No solution specified') | 610 raise gclient_utils.Error('No solution specified') |
| 580 revision_overrides = self._EnforceRevisions() | 611 revision_overrides = self._EnforceRevisions() |
| 581 | 612 pm = None |
| 582 # When running runhooks --force, there's no need to consult the SCM. | |
| 583 # All known hooks are expected to run unconditionally regardless of working | |
| 584 # copy state, so skip the SCM status check. | |
| 585 run_scm = not (command == 'runhooks' and self._options.force) | |
| 586 | |
| 587 entries = {} | |
| 588 file_list = [] | |
| 589 # Run on the base solutions first. | |
| 590 for solution in self.dependencies: | |
| 591 name = solution.name | |
| 592 if name in entries: | |
| 593 raise gclient_utils.Error("solution %s specified more than once" % name) | |
| 594 url = solution.url | |
| 595 entries[name] = url | |
| 596 if run_scm and url: | |
| 597 self._options.revision = revision_overrides.get(name) | |
| 598 scm = gclient_scm.CreateSCM(url, self.root_dir(), name) | |
| 599 scm.RunCommand(command, self._options, args, file_list) | |
| 600 file_list = [os.path.join(name, f.strip()) for f in file_list] | |
| 601 self._options.revision = None | |
| 602 | |
| 603 # Process the dependencies next (sort alphanumerically to ensure that | |
| 604 # containing directories get populated first and for readability) | |
| 605 deps = self._ParseAllDeps(entries) | |
| 606 deps_to_process = deps.keys() | |
| 607 deps_to_process.sort() | |
| 608 | |
| 609 # First pass for direct dependencies. | |
| 610 if command == 'update' and not self._options.verbose: | 613 if command == 'update' and not self._options.verbose: |
| 611 pm = Progress('Syncing projects', len(deps_to_process)) | 614 pm = Progress('Syncing projects', len(self.tree(False)) + 1) |
| 612 for d in deps_to_process: | 615 self.RunCommandRecursively(self._options, revision_overrides, |
| 613 if command == 'update' and not self._options.verbose: | 616 command, args, pm) |
| 614 pm.update() | 617 if pm: |
| 615 if type(deps[d]) == str: | |
| 616 url = deps[d] | |
| 617 entries[d] = url | |
| 618 if run_scm: | |
| 619 self._options.revision = revision_overrides.get(d) | |
| 620 scm = gclient_scm.CreateSCM(url, self.root_dir(), d) | |
| 621 scm.RunCommand(command, self._options, args, file_list) | |
| 622 self._options.revision = None | |
| 623 elif isinstance(deps[d], self.FileImpl): | |
| 624 file_dep = deps[d] | |
| 625 self._options.revision = file_dep.GetRevision() | |
| 626 if run_scm: | |
| 627 scm = gclient_scm.CreateSCM(file_dep.GetPath(), self.root_dir(), d) | |
| 628 scm.RunCommand("updatesingle", self._options, | |
| 629 args + [file_dep.GetFilename()], file_list) | |
| 630 | |
| 631 if command == 'update' and not self._options.verbose: | |
| 632 pm.end() | 618 pm.end() |
| 633 | 619 |
| 634 # Second pass for inherited deps (via the From keyword) | 620 # Once all the dependencies have been processed, it's now safe to run the |
| 635 for d in deps_to_process: | 621 # hooks. |
| 636 if isinstance(deps[d], self.FromImpl): | 622 if not self._options.nohooks: |
| 637 # Getting the URL from the sub_deps file can involve having to resolve | 623 self.RunHooksRecursively(self._options) |
| 638 # a File() or having to resolve a relative URL. To resolve relative | |
| 639 # URLs, we need to pass in the orignal sub deps URL. | |
| 640 sub_deps_base_url = deps[deps[d].module_name] | |
| 641 sub_deps = Dependency(self, deps[d].module_name, sub_deps_base_url | |
| 642 ).ParseDepsFile(False) | |
| 643 url = deps[d].GetUrl(d, sub_deps_base_url, self.root_dir(), sub_deps) | |
| 644 entries[d] = url | |
| 645 if run_scm: | |
| 646 self._options.revision = revision_overrides.get(d) | |
| 647 scm = gclient_scm.CreateSCM(url, self.root_dir(), d) | |
| 648 scm.RunCommand(command, self._options, args, file_list) | |
| 649 self._options.revision = None | |
| 650 | |
| 651 # Convert all absolute paths to relative. | |
| 652 for i in range(len(file_list)): | |
| 653 # TODO(phajdan.jr): We should know exactly when the paths are absolute. | |
| 654 # It depends on the command being executed (like runhooks vs sync). | |
| 655 if not os.path.isabs(file_list[i]): | |
| 656 continue | |
| 657 | |
| 658 prefix = os.path.commonprefix([self.root_dir().lower(), | |
| 659 file_list[i].lower()]) | |
| 660 file_list[i] = file_list[i][len(prefix):] | |
| 661 | |
| 662 # Strip any leading path separators. | |
| 663 while file_list[i].startswith('\\') or file_list[i].startswith('/'): | |
| 664 file_list[i] = file_list[i][1:] | |
| 665 | |
| 666 is_using_git = gclient_utils.IsUsingGit(self.root_dir(), entries.keys()) | |
| 667 self._RunHooks(command, file_list, is_using_git) | |
| 668 | 624 |
| 669 if command == 'update': | 625 if command == 'update': |
| 670 # Notify the user if there is an orphaned entry in their working copy. | 626 # Notify the user if there is an orphaned entry in their working copy. |
| 671 # Only delete the directory if there are no changes in it, and | 627 # Only delete the directory if there are no changes in it, and |
| 672 # delete_unversioned_trees is set to true. | 628 # delete_unversioned_trees is set to true. |
| 673 prev_entries = self._ReadEntries() | 629 entries = [i.name for i in self.tree(False)] |
| 674 for entry in prev_entries: | 630 for entry, prev_url in self._ReadEntries().iteritems(): |
| 675 # Fix path separator on Windows. | 631 # Fix path separator on Windows. |
| 676 entry_fixed = entry.replace('/', os.path.sep) | 632 entry_fixed = entry.replace('/', os.path.sep) |
| 677 e_dir = os.path.join(self.root_dir(), entry_fixed) | 633 e_dir = os.path.join(self.root_dir(), entry_fixed) |
| 678 # Use entry and not entry_fixed there. | 634 # Use entry and not entry_fixed there. |
| 679 if entry not in entries and os.path.exists(e_dir): | 635 if entry not in entries and os.path.exists(e_dir): |
| 680 modified_files = False | 636 file_list = [] |
| 681 if isinstance(prev_entries, list): | 637 scm = gclient_scm.CreateSCM(prev_url, self.root_dir(), entry_fixed) |
| 682 # old .gclient_entries format was list, now dict | 638 scm.status(self._options, [], file_list) |
| 683 modified_files = gclient_scm.scm.SVN.CaptureStatus(e_dir) | 639 modified_files = file_list != [] |
| 684 else: | |
| 685 file_list = [] | |
| 686 scm = gclient_scm.CreateSCM(prev_entries[entry], self.root_dir(), | |
| 687 entry_fixed) | |
| 688 scm.status(self._options, [], file_list) | |
| 689 modified_files = file_list != [] | |
| 690 if not self._options.delete_unversioned_trees or modified_files: | 640 if not self._options.delete_unversioned_trees or modified_files: |
| 691 # There are modified files in this entry. Keep warning until | 641 # There are modified files in this entry. Keep warning until |
| 692 # removed. | 642 # removed. |
| 693 print(('\nWARNING: \'%s\' is no longer part of this client. ' | 643 print(('\nWARNING: \'%s\' is no longer part of this client. ' |
| 694 'It is recommended that you manually remove it.\n') % | 644 'It is recommended that you manually remove it.\n') % |
| 695 entry_fixed) | 645 entry_fixed) |
| 696 else: | 646 else: |
| 697 # Delete the entry | 647 # Delete the entry |
| 698 print('\n________ deleting \'%s\' in \'%s\'' % ( | 648 print('\n________ deleting \'%s\' in \'%s\'' % ( |
| 699 entry_fixed, self.root_dir())) | 649 entry_fixed, self.root_dir())) |
| 700 gclient_utils.RemoveDirectory(e_dir) | 650 gclient_utils.RemoveDirectory(e_dir) |
| 701 # record the current list of entries for next time | 651 # record the current list of entries for next time |
| 702 self._SaveEntries(entries) | 652 self._SaveEntries() |
| 703 return 0 | 653 return 0 |
| 704 | 654 |
| 705 def PrintRevInfo(self): | 655 def PrintRevInfo(self): |
| 706 """Output revision info mapping for the client and its dependencies. | 656 """Output revision info mapping for the client and its dependencies. |
| 707 | 657 |
| 708 This allows the capture of an overall "revision" for the source tree that | 658 This allows the capture of an overall "revision" for the source tree that |
| 709 can be used to reproduce the same tree in the future. It is only useful for | 659 can be used to reproduce the same tree in the future. It is only useful for |
| 710 "unpinned dependencies", i.e. DEPS/deps references without a svn revision | 660 "unpinned dependencies", i.e. DEPS/deps references without a svn revision |
| 711 number or a git hash. A git branch name isn't "pinned" since the actual | 661 number or a git hash. A git branch name isn't "pinned" since the actual |
| 712 commit can change. | 662 commit can change. |
| 713 | 663 |
| 714 The --snapshot option allows creating a .gclient file to reproduce the tree. | 664 The --snapshot option allows creating a .gclient file to reproduce the tree. |
| 715 """ | 665 """ |
| 716 if not self.dependencies: | 666 if not self.dependencies: |
| 717 raise gclient_utils.Error('No solution specified') | 667 raise gclient_utils.Error('No solution specified') |
| 668 # Load all the settings. |
| 669 self.RunCommandRecursively(self._options, {}, None, [], None) |
| 718 | 670 |
| 719 # Inner helper to generate base url and rev tuple | |
| 720 def GetURLAndRev(name, original_url): | 671 def GetURLAndRev(name, original_url): |
| 672 """Returns the revision-qualified SCM url.""" |
| 721 url, _ = gclient_utils.SplitUrlRevision(original_url) | 673 url, _ = gclient_utils.SplitUrlRevision(original_url) |
| 722 scm = gclient_scm.CreateSCM(original_url, self.root_dir(), name) | 674 scm = gclient_scm.CreateSCM(original_url, self.root_dir(), name) |
| 723 return (url, scm.revinfo(self._options, [], None)) | 675 if not os.path.isdir(scm.checkout_path): |
| 676 return None |
| 677 return '%s@%s' % (url, scm.revinfo(self._options, [], None)) |
| 724 | 678 |
| 725 # text of the snapshot gclient file | |
| 726 new_gclient = "" | |
| 727 # Dictionary of { path : SCM url } to ensure no duplicate solutions | |
| 728 solution_names = {} | |
| 729 entries = {} | |
| 730 # Run on the base solutions first. | |
| 731 for solution in self.dependencies: | |
| 732 # Dictionary of { path : SCM url } to describe the gclient checkout | |
| 733 name = solution.name | |
| 734 if name in solution_names: | |
| 735 raise gclient_utils.Error("solution %s specified more than once" % name) | |
| 736 (url, rev) = GetURLAndRev(name, solution.url) | |
| 737 entries[name] = "%s@%s" % (url, rev) | |
| 738 solution_names[name] = "%s@%s" % (url, rev) | |
| 739 | |
| 740 # Process the dependencies next (sort alphanumerically to ensure that | |
| 741 # containing directories get populated first and for readability) | |
| 742 deps = self._ParseAllDeps(entries) | |
| 743 deps_to_process = deps.keys() | |
| 744 deps_to_process.sort() | |
| 745 | |
| 746 # First pass for direct dependencies. | |
| 747 for d in deps_to_process: | |
| 748 if type(deps[d]) == str: | |
| 749 (url, rev) = GetURLAndRev(d, deps[d]) | |
| 750 entries[d] = "%s@%s" % (url, rev) | |
| 751 | |
| 752 # Second pass for inherited deps (via the From keyword) | |
| 753 for d in deps_to_process: | |
| 754 if isinstance(deps[d], self.FromImpl): | |
| 755 deps_parent_url = entries[deps[d].module_name] | |
| 756 if deps_parent_url.find("@") < 0: | |
| 757 raise gclient_utils.Error("From %s missing revisioned url" % | |
| 758 deps[d].module_name) | |
| 759 sub_deps_base_url = deps[deps[d].module_name] | |
| 760 sub_deps = Dependency(self, deps[d].module_name, sub_deps_base_url | |
| 761 ).ParseDepsFile(False) | |
| 762 url = deps[d].GetUrl(d, sub_deps_base_url, self.root_dir(), sub_deps) | |
| 763 (url, rev) = GetURLAndRev(d, url) | |
| 764 entries[d] = "%s@%s" % (url, rev) | |
| 765 | |
| 766 # Build the snapshot configuration string | |
| 767 if self._options.snapshot: | 679 if self._options.snapshot: |
| 768 url = entries.pop(name) | 680 new_gclient = '' |
| 769 custom_deps = ''.join([' \"%s\": \"%s\",\n' % (x, entries[x]) | 681 # First level at .gclient |
| 770 for x in sorted(entries.keys())]) | 682 for d in self.dependencies: |
| 771 | 683 entries = {} |
| 772 new_gclient += self.DEFAULT_SNAPSHOT_SOLUTION_TEXT % { | 684 def GrabDeps(sol): |
| 773 'solution_name': name, | 685 """Recursively grab dependencies.""" |
| 774 'solution_url': url, | 686 for i in sol.dependencies: |
| 775 'safesync_url' : '', | 687 entries[i.name] = GetURLAndRev(i.name, i.parsed_url) |
| 776 'solution_deps': custom_deps, | 688 GrabDeps(i) |
| 777 } | 689 GrabDeps(d) |
| 690 custom_deps = [] |
| 691 for k in sorted(entries.keys()): |
| 692 if entries[k]: |
| 693 # Quotes aren't escaped... |
| 694 custom_deps.append(' \"%s\": \'%s\',\n' % (k, entries[k])) |
| 695 else: |
| 696 custom_deps.append(' \"%s\": None,\n' % k) |
| 697 new_gclient += self.DEFAULT_SNAPSHOT_SOLUTION_TEXT % { |
| 698 'solution_name': d.name, |
| 699 'solution_url': d.url, |
| 700 'safesync_url' : d.safesync_url or '', |
| 701 'solution_deps': ''.join(custom_deps), |
| 702 } |
| 703 # Print the snapshot configuration file |
| 704 print(self.DEFAULT_SNAPSHOT_FILE_TEXT % {'solution_list': new_gclient}) |
| 778 else: | 705 else: |
| 779 print(';\n'.join(['%s: %s' % (x, entries[x]) | 706 entries = sorted(self.tree(False), key=lambda i: i.name) |
| 780 for x in sorted(entries.keys())])) | 707 for entry in entries: |
| 781 | 708 revision = GetURLAndRev(entry.name, entry.parsed_url) |
| 782 # Print the snapshot configuration file | 709 line = '%s: %s' % (entry.name, revision) |
| 783 if self._options.snapshot: | 710 if not entry is entries[-1]: |
| 784 config = self.DEFAULT_SNAPSHOT_FILE_TEXT % {'solution_list': new_gclient} | 711 line += ';' |
| 785 snapclient = GClient(self.root_dir(), self._options) | 712 print line |
| 786 snapclient.SetConfig(config) | |
| 787 print(snapclient.config_content) | |
| 788 | 713 |
| 789 def ParseDepsFile(self, direct_reference): | 714 def ParseDepsFile(self, direct_reference): |
| 790 """No DEPS to parse for a .gclient file.""" | 715 """No DEPS to parse for a .gclient file.""" |
| 791 self.direct_reference = direct_reference | 716 self.direct_reference = direct_reference |
| 792 self.deps_parsed = True | 717 self.deps_parsed = True |
| 793 | 718 |
| 794 def root_dir(self): | 719 def root_dir(self): |
| 795 """Root directory of gclient checkout.""" | 720 """Root directory of gclient checkout.""" |
| 796 return self._root_dir | 721 return self._root_dir |
| 797 | 722 |
| (...skipping 300 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1098 'unpinned dependencies', i.e. DEPS/deps references without a svn revision | 1023 'unpinned dependencies', i.e. DEPS/deps references without a svn revision |
| 1099 number or a git hash. A git branch name isn't 'pinned' since the actual | 1024 number or a git hash. A git branch name isn't 'pinned' since the actual |
| 1100 commit can change. | 1025 commit can change. |
| 1101 """ | 1026 """ |
| 1102 parser.add_option('--deps', dest='deps_os', metavar='OS_LIST', | 1027 parser.add_option('--deps', dest='deps_os', metavar='OS_LIST', |
| 1103 help='override deps for the specified (comma-separated) ' | 1028 help='override deps for the specified (comma-separated) ' |
| 1104 'platform(s); \'all\' will process all deps_os ' | 1029 'platform(s); \'all\' will process all deps_os ' |
| 1105 'references') | 1030 'references') |
| 1106 parser.add_option('-s', '--snapshot', action='store_true', | 1031 parser.add_option('-s', '--snapshot', action='store_true', |
| 1107 help='creates a snapshot .gclient file of the current ' | 1032 help='creates a snapshot .gclient file of the current ' |
| 1108 'version of all repositories to reproduce the tree, ' | 1033 'version of all repositories to reproduce the tree') |
| 1109 'implies -a') | |
| 1110 (options, args) = parser.parse_args(args) | 1034 (options, args) = parser.parse_args(args) |
| 1111 client = GClient.LoadCurrentConfig(options) | 1035 client = GClient.LoadCurrentConfig(options) |
| 1112 if not client: | 1036 if not client: |
| 1113 raise gclient_utils.Error('client not configured; see \'gclient config\'') | 1037 raise gclient_utils.Error('client not configured; see \'gclient config\'') |
| 1114 client.PrintRevInfo() | 1038 client.PrintRevInfo() |
| 1115 return 0 | 1039 return 0 |
| 1116 | 1040 |
| 1117 | 1041 |
| 1118 def Command(name): | 1042 def Command(name): |
| 1119 return getattr(sys.modules[__name__], 'CMD' + name, None) | 1043 return getattr(sys.modules[__name__], 'CMD' + name, None) |
| (...skipping 71 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1191 return CMDhelp(parser, argv) | 1115 return CMDhelp(parser, argv) |
| 1192 except gclient_utils.Error, e: | 1116 except gclient_utils.Error, e: |
| 1193 print >> sys.stderr, 'Error: %s' % str(e) | 1117 print >> sys.stderr, 'Error: %s' % str(e) |
| 1194 return 1 | 1118 return 1 |
| 1195 | 1119 |
| 1196 | 1120 |
| 1197 if '__main__' == __name__: | 1121 if '__main__' == __name__: |
| 1198 sys.exit(Main(sys.argv[1:])) | 1122 sys.exit(Main(sys.argv[1:])) |
| 1199 | 1123 |
| 1200 # vim: ts=2:sw=2:tw=80:et: | 1124 # vim: ts=2:sw=2:tw=80:et: |
| OLD | NEW |