OLD | NEW |
1 #!/usr/bin/python | 1 #!/usr/bin/python |
2 # Copyright (c) 2010 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2010 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """Meta checkout manager supporting both Subversion and GIT. | 6 """Meta checkout manager supporting both Subversion and GIT. |
7 | 7 |
8 Files | 8 Files |
9 .gclient : Current client configuration, written by 'config' command. | 9 .gclient : Current client configuration, written by 'config' command. |
10 Format is a Python script defining 'solutions', a list whose | 10 Format is a Python script defining 'solutions', a list whose |
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
42 it will be removed from the list and the list will be extended | 42 it will be removed from the list and the list will be extended |
43 by the list of matching files. | 43 by the list of matching files. |
44 | 44 |
45 Example: | 45 Example: |
46 hooks = [ | 46 hooks = [ |
47 { "pattern": "\\.(gif|jpe?g|pr0n|png)$", | 47 { "pattern": "\\.(gif|jpe?g|pr0n|png)$", |
48 "action": ["python", "image_indexer.py", "--all"]}, | 48 "action": ["python", "image_indexer.py", "--all"]}, |
49 ] | 49 ] |
50 """ | 50 """ |
51 | 51 |
52 __version__ = "0.4.1" | 52 __version__ = "0.5" |
53 | 53 |
54 import errno | |
55 import logging | 54 import logging |
56 import optparse | 55 import optparse |
57 import os | 56 import os |
58 import pprint | 57 import pprint |
59 import re | 58 import re |
60 import subprocess | 59 import subprocess |
61 import sys | 60 import sys |
62 import urlparse | 61 import urlparse |
63 import urllib | 62 import urllib |
64 | 63 |
(...skipping 25 matching lines...) Expand all Loading... |
90 | 89 |
91 sub_target_name is an optional parameter if the module name in the other | 90 sub_target_name is an optional parameter if the module name in the other |
92 DEPS file is different. E.g., you might want to map src/net to net.""" | 91 DEPS file is different. E.g., you might want to map src/net to net.""" |
93 self.module_name = module_name | 92 self.module_name = module_name |
94 self.sub_target_name = sub_target_name | 93 self.sub_target_name = sub_target_name |
95 | 94 |
96 def __str__(self): | 95 def __str__(self): |
97 return 'From(%s, %s)' % (repr(self.module_name), | 96 return 'From(%s, %s)' % (repr(self.module_name), |
98 repr(self.sub_target_name)) | 97 repr(self.sub_target_name)) |
99 | 98 |
100 def GetUrl(self, target_name, sub_deps_base_url, root_dir, sub_deps): | |
101 """Resolve the URL for this From entry.""" | |
102 sub_deps_target_name = target_name | |
103 if self.sub_target_name: | |
104 sub_deps_target_name = self.sub_target_name | |
105 url = sub_deps[sub_deps_target_name] | |
106 if url.startswith('/'): | |
107 # If it's a relative URL, we need to resolve the URL relative to the | |
108 # sub deps base URL. | |
109 if not isinstance(sub_deps_base_url, basestring): | |
110 sub_deps_base_url = sub_deps_base_url.GetPath() | |
111 scm = gclient_scm.CreateSCM(sub_deps_base_url, root_dir, | |
112 None) | |
113 url = scm.FullUrlForRelativeUrl(url) | |
114 return url | |
115 | |
116 class FileImpl(object): | 99 class FileImpl(object): |
117 """Used to implement the File('') syntax which lets you sync a single file | 100 """Used to implement the File('') syntax which lets you sync a single file |
118 from an SVN repo.""" | 101 from an SVN repo.""" |
119 | 102 |
120 def __init__(self, file_location): | 103 def __init__(self, file_location): |
121 self.file_location = file_location | 104 self.file_location = file_location |
122 | 105 |
123 def __str__(self): | 106 def __str__(self): |
124 return 'File("%s")' % self.file_location | 107 return 'File("%s")' % self.file_location |
125 | 108 |
(...skipping 27 matching lines...) Expand all Loading... |
153 class Dependency(GClientKeywords): | 136 class Dependency(GClientKeywords): |
154 """Object that represents a dependency checkout.""" | 137 """Object that represents a dependency checkout.""" |
155 DEPS_FILE = 'DEPS' | 138 DEPS_FILE = 'DEPS' |
156 | 139 |
157 def __init__(self, parent, name, url, safesync_url=None, custom_deps=None, | 140 def __init__(self, parent, name, url, safesync_url=None, custom_deps=None, |
158 custom_vars=None, deps_file=None): | 141 custom_vars=None, deps_file=None): |
159 GClientKeywords.__init__(self) | 142 GClientKeywords.__init__(self) |
160 self.parent = parent | 143 self.parent = parent |
161 self.name = name | 144 self.name = name |
162 self.url = url | 145 self.url = url |
| 146 self.parsed_url = None |
163 # These 2 are only set in .gclient and not in DEPS files. | 147 # These 2 are only set in .gclient and not in DEPS files. |
164 self.safesync_url = safesync_url | 148 self.safesync_url = safesync_url |
165 self.custom_vars = custom_vars or {} | 149 self.custom_vars = custom_vars or {} |
166 self.custom_deps = custom_deps or {} | 150 self.custom_deps = custom_deps or {} |
167 self.deps_hooks = [] | 151 self.deps_hooks = [] |
168 self.dependencies = [] | 152 self.dependencies = [] |
169 self.deps_file = deps_file or self.DEPS_FILE | 153 self.deps_file = deps_file or self.DEPS_FILE |
| 154 # A cache of the files affected by the current operation, necessary for |
| 155 # hooks. |
| 156 self.file_list = [] |
170 self.deps_parsed = False | 157 self.deps_parsed = False |
171 self.direct_reference = False | 158 self.direct_reference = False |
172 | 159 |
173 # Sanity checks | 160 # Sanity checks |
174 if not self.name and self.parent: | 161 if not self.name and self.parent: |
175 raise gclient_utils.Error('Dependency without name') | 162 raise gclient_utils.Error('Dependency without name') |
| 163 if self.name in [d.name for d in self.tree(False)]: |
| 164 raise gclient_utils.Error('Dependency %s specified more than once' % |
| 165 self.name) |
176 if not isinstance(self.url, | 166 if not isinstance(self.url, |
177 (basestring, self.FromImpl, self.FileImpl, None.__class__)): | 167 (basestring, self.FromImpl, self.FileImpl, None.__class__)): |
178 raise gclient_utils.Error('dependency url must be either a string, None, ' | 168 raise gclient_utils.Error('dependency url must be either a string, None, ' |
179 'File() or From() instead of %s' % | 169 'File() or From() instead of %s' % |
180 self.url.__class__.__name__) | 170 self.url.__class__.__name__) |
181 if '/' in self.deps_file or '\\' in self.deps_file: | 171 if '/' in self.deps_file or '\\' in self.deps_file: |
182 raise gclient_utils.Error('deps_file name must not be a path, just a ' | 172 raise gclient_utils.Error('deps_file name must not be a path, just a ' |
183 'filename. %s' % self.deps_file) | 173 'filename. %s' % self.deps_file) |
184 | 174 |
| 175 def LateOverride(self, url): |
| 176 overriden_url = self.get_custom_deps(self.name, url) |
| 177 if overriden_url != url: |
| 178 self.parsed_url = overriden_url |
| 179 logging.debug('%s, %s was overriden to %s' % (self.name, url, |
| 180 self.parsed_url)) |
| 181 elif isinstance(url, self.FromImpl): |
| 182 ref = [dep for dep in self.tree(True) if url.module_name == dep.name] |
| 183 if not len(ref) == 1: |
| 184 raise Exception('Failed to find one reference to %s. %s' % ( |
| 185 url.module_name, ref)) |
| 186 ref = ref[0] |
| 187 sub_target = url.sub_target_name or url |
| 188 # Make sure the referenced dependency DEPS file is loaded and file the |
| 189 # inner referenced dependency. |
| 190 ref.ParseDepsFile(False) |
| 191 found_dep = None |
| 192 for d in ref.dependencies: |
| 193 if d.name == sub_target: |
| 194 found_dep = d |
| 195 break |
| 196 if not found_dep: |
| 197 raise Exception('Couldn\'t find %s in %s, referenced by %s' % ( |
| 198 sub_target, ref.name, self.name)) |
| 199 # Call LateOverride() again. |
| 200 self.parsed_url = found_dep.LateOverride(found_dep.url) |
| 201 logging.debug('%s, %s to %s' % (self.name, url, self.parsed_url)) |
| 202 elif isinstance(url, basestring): |
| 203 parsed_url = urlparse.urlparse(url) |
| 204 if not parsed_url[0]: |
| 205 # A relative url. Fetch the real base. |
| 206 path = parsed_url[2] |
| 207 if not path.startswith('/'): |
| 208 raise gclient_utils.Error( |
| 209 'relative DEPS entry \'%s\' must begin with a slash' % url) |
| 210 # Create a scm just to query the full url. |
| 211 scm = gclient_scm.CreateSCM(self.parent.parsed_url, self.root_dir(), |
| 212 None) |
| 213 self.parsed_url = scm.FullUrlForRelativeUrl(url) |
| 214 else: |
| 215 self.parsed_url = url |
| 216 logging.debug('%s, %s -> %s' % (self.name, url, self.parsed_url)) |
| 217 elif isinstance(url, self.FileImpl): |
| 218 self.parsed_url = url |
| 219 logging.debug('%s, %s -> %s (File)' % (self.name, url, self.parsed_url)) |
| 220 return self.parsed_url |
| 221 |
185 def ParseDepsFile(self, direct_reference): | 222 def ParseDepsFile(self, direct_reference): |
186 """Parses the DEPS file for this dependency.""" | 223 """Parses the DEPS file for this dependency.""" |
187 if direct_reference: | 224 if direct_reference: |
188 # Maybe it was referenced earlier by a From() keyword but it's now | 225 # Maybe it was referenced earlier by a From() keyword but it's now |
189 # directly referenced. | 226 # directly referenced. |
190 self.direct_reference = direct_reference | 227 self.direct_reference = direct_reference |
| 228 if self.deps_parsed: |
| 229 return |
191 self.deps_parsed = True | 230 self.deps_parsed = True |
192 filepath = os.path.join(self.root_dir(), self.name, self.deps_file) | 231 filepath = os.path.join(self.root_dir(), self.name, self.deps_file) |
193 if not os.path.isfile(filepath): | 232 if not os.path.isfile(filepath): |
194 return {} | 233 return |
195 deps_content = gclient_utils.FileRead(filepath) | 234 deps_content = gclient_utils.FileRead(filepath) |
196 | 235 |
197 # Eval the content. | 236 # Eval the content. |
198 # One thing is unintuitive, vars= {} must happen before Var() use. | 237 # One thing is unintuitive, vars= {} must happen before Var() use. |
199 local_scope = {} | 238 local_scope = {} |
200 var = self.VarImpl(self.custom_vars, local_scope) | 239 var = self.VarImpl(self.custom_vars, local_scope) |
201 global_scope = { | 240 global_scope = { |
202 'File': self.FileImpl, | 241 'File': self.FileImpl, |
203 'From': self.FromImpl, | 242 'From': self.FromImpl, |
204 'Var': var.Lookup, | 243 'Var': var.Lookup, |
(...skipping 30 matching lines...) Expand all Loading... |
235 # the dictionary using paths relative to the directory containing | 274 # the dictionary using paths relative to the directory containing |
236 # the DEPS file. | 275 # the DEPS file. |
237 use_relative_paths = local_scope.get('use_relative_paths', False) | 276 use_relative_paths = local_scope.get('use_relative_paths', False) |
238 if use_relative_paths: | 277 if use_relative_paths: |
239 rel_deps = {} | 278 rel_deps = {} |
240 for d, url in deps.items(): | 279 for d, url in deps.items(): |
241 # normpath is required to allow DEPS to use .. in their | 280 # normpath is required to allow DEPS to use .. in their |
242 # dependency local path. | 281 # dependency local path. |
243 rel_deps[os.path.normpath(os.path.join(self.name, d))] = url | 282 rel_deps[os.path.normpath(os.path.join(self.name, d))] = url |
244 deps = rel_deps | 283 deps = rel_deps |
245 # TODO(maruel): Add these dependencies into self.dependencies. | |
246 return deps | |
247 | 284 |
248 def _ParseAllDeps(self, solution_urls): | 285 # Convert the deps into real Dependency. |
249 """Parse the complete list of dependencies for the client. | 286 for name, url in deps.iteritems(): |
| 287 if name in [s.name for s in self.dependencies]: |
| 288 raise |
| 289 self.dependencies.append(Dependency(self, name, url)) |
| 290 # Sort by name. |
| 291 self.dependencies.sort(key=lambda x: x.name) |
| 292 logging.info('Loaded: %s' % str(self)) |
250 | 293 |
251 Args: | 294 def RunCommandRecursively(self, options, revision_overrides, |
252 solution_urls: A dict mapping module names (as relative paths) to URLs | 295 command, args, pm): |
253 corresponding to the solutions specified by the client. This parameter | 296 """Runs 'command' before parsing the DEPS in case it's a initial checkout |
254 is passed as an optimization. | 297 or a revert.""" |
| 298 assert self.file_list == [] |
| 299 # When running runhooks, there's no need to consult the SCM. |
| 300 # All known hooks are expected to run unconditionally regardless of working |
| 301 # copy state, so skip the SCM status check. |
| 302 run_scm = command not in ('runhooks', None) |
| 303 self.LateOverride(self.url) |
| 304 if run_scm and self.parsed_url: |
| 305 if isinstance(self.parsed_url, self.FileImpl): |
| 306 # Special support for single-file checkout. |
| 307 options.revision = self.parsed_url.GetRevision() |
| 308 scm = gclient_scm.CreateSCM(self.parsed_url.GetPath(), self.root_dir(), |
| 309 self.name) |
| 310 scm.RunCommand("updatesingle", options, |
| 311 args + [self.parsed_url.GetFilename()], self.file_list) |
| 312 else: |
| 313 options.revision = revision_overrides.get(self.name) |
| 314 scm = gclient_scm.CreateSCM(self.parsed_url, self.root_dir(), self.name) |
| 315 scm.RunCommand(command, options, args, self.file_list) |
| 316 self.file_list = [os.path.join(self.name, f.strip()) |
| 317 for f in self.file_list] |
| 318 options.revision = None |
| 319 if pm: |
| 320 # The + 1 comes from the fact that .gclient is considered a step in |
| 321 # itself, .i.e. this code is called one time for the .gclient. This is not |
| 322 # conceptually correct but it simplifies code. |
| 323 pm._total = len(self.tree(False)) + 1 |
| 324 pm.update() |
| 325 if self.recursion_limit(): |
| 326 # Then we can parse the DEPS file. |
| 327 self.ParseDepsFile(True) |
| 328 if pm: |
| 329 pm._total = len(self.tree(False)) + 1 |
| 330 pm.update(0) |
| 331 # Parse the dependencies of this dependency. |
| 332 for s in self.dependencies: |
| 333 # TODO(maruel): All these can run concurrently! No need for threads, |
| 334 # just buffer stdout&stderr on pipes and flush as they complete. |
| 335 # Watch out for stdin. |
| 336 s.RunCommandRecursively(options, revision_overrides, command, args, pm) |
255 | 337 |
256 Returns: | 338 def RunHooksRecursively(self, options): |
257 A dict mapping module names (as relative paths) to URLs corresponding | 339 """Evaluates all hooks, running actions as needed. RunCommandRecursively() |
258 to the entire set of dependencies to checkout for the given client. | 340 must have been called before to load the DEPS.""" |
| 341 # If "--force" was specified, run all hooks regardless of what files have |
| 342 # changed. |
| 343 if self.deps_hooks: |
| 344 # TODO(maruel): If the user is using git or git-svn, then we don't know |
| 345 # what files have changed so we always run all hooks. It'd be nice to fix |
| 346 # that. |
| 347 if (options.force or |
| 348 gclient_scm.GetScmName(self.parsed_url) in ('git', None) or |
| 349 os.path.isdir(os.path.join(self.root_dir(), self.name, '.git'))): |
| 350 for hook_dict in self.deps_hooks: |
| 351 self._RunHookAction(hook_dict, []) |
| 352 else: |
| 353 # TODO(phajdan.jr): We should know exactly when the paths are absolute. |
| 354 # Convert all absolute paths to relative. |
| 355 for i in range(len(self.file_list)): |
| 356 # It depends on the command being executed (like runhooks vs sync). |
| 357 if not os.path.isabs(self.file_list[i]): |
| 358 continue |
259 | 359 |
260 Raises: | 360 prefix = os.path.commonprefix([self.root_dir().lower(), |
261 Error: If a dependency conflicts with another dependency or of a solution. | 361 self.file_list[i].lower()]) |
262 """ | 362 self.file_list[i] = self.file_list[i][len(prefix):] |
263 deps = {} | |
264 for solution in self.dependencies: | |
265 solution_deps = solution.ParseDepsFile(True) | |
266 | 363 |
267 # If a line is in custom_deps, but not in the solution, we want to append | 364 # Strip any leading path separators. |
268 # this line to the solution. | 365 while file_list[i].startswith('\\') or file_list[i].startswith('/'): |
269 for d in solution.custom_deps: | 366 self.file_list[i] = self.file_list[i][1:] |
270 if d not in solution_deps: | |
271 solution_deps[d] = solution.custom_deps[d] | |
272 | 367 |
273 for d in solution_deps: | 368 # Run hooks on the basis of whether the files from the gclient operation |
274 if d in solution.custom_deps: | 369 # match each hook's pattern. |
275 # Dependency is overriden. | 370 for hook_dict in self.deps_hooks: |
276 url = solution.custom_deps[d] | 371 pattern = re.compile(hook_dict['pattern']) |
277 if url is None: | 372 matching_file_list = [f for f in self.file_list if pattern.search(f)] |
278 continue | 373 if matching_file_list: |
279 else: | 374 self._RunHookAction(hook_dict, matching_file_list) |
280 url = solution_deps[d] | 375 if self.recursion_limit(): |
281 # if we have a From reference dependent on another solution, then | 376 for s in self.dependencies: |
282 # just skip the From reference. When we pull deps for the solution, | 377 s.RunHooksRecursively(options) |
283 # we will take care of this dependency. | |
284 # | |
285 # If multiple solutions all have the same From reference, then we | |
286 # should only add one to our list of dependencies. | |
287 if isinstance(url, self.FromImpl): | |
288 if url.module_name in solution_urls: | |
289 # Already parsed. | |
290 continue | |
291 if d in deps and type(deps[d]) != str: | |
292 if url.module_name == deps[d].module_name: | |
293 continue | |
294 elif isinstance(url, str): | |
295 parsed_url = urlparse.urlparse(url) | |
296 scheme = parsed_url[0] | |
297 if not scheme: | |
298 # A relative url. Fetch the real base. | |
299 path = parsed_url[2] | |
300 if path[0] != "/": | |
301 raise gclient_utils.Error( | |
302 "relative DEPS entry \"%s\" must begin with a slash" % d) | |
303 # Create a scm just to query the full url. | |
304 scm = gclient_scm.CreateSCM(solution.url, self.root_dir(), | |
305 None) | |
306 url = scm.FullUrlForRelativeUrl(url) | |
307 if d in deps and deps[d] != url: | |
308 raise gclient_utils.Error( | |
309 "Solutions have conflicting versions of dependency \"%s\"" % d) | |
310 if d in solution_urls and solution_urls[d] != url: | |
311 raise gclient_utils.Error( | |
312 "Dependency \"%s\" conflicts with specified solution" % d) | |
313 # Grab the dependency. | |
314 deps[d] = url | |
315 return deps | |
316 | |
317 def _RunHooks(self, command, file_list, is_using_git): | |
318 """Evaluates all hooks, running actions as needed. | |
319 """ | |
320 # Hooks only run for these command types. | |
321 if not command in ('update', 'revert', 'runhooks'): | |
322 return | |
323 | |
324 # Hooks only run when --nohooks is not specified | |
325 if self._options.nohooks: | |
326 return | |
327 | |
328 # Get any hooks from the .gclient file. | |
329 hooks = self.deps_hooks[:] | |
330 # Add any hooks found in DEPS files. | |
331 for d in self.dependencies: | |
332 hooks.extend(d.deps_hooks) | |
333 | |
334 # If "--force" was specified, run all hooks regardless of what files have | |
335 # changed. If the user is using git, then we don't know what files have | |
336 # changed so we always run all hooks. | |
337 if self._options.force or is_using_git: | |
338 for hook_dict in hooks: | |
339 self._RunHookAction(hook_dict, []) | |
340 return | |
341 | |
342 # Run hooks on the basis of whether the files from the gclient operation | |
343 # match each hook's pattern. | |
344 for hook_dict in hooks: | |
345 pattern = re.compile(hook_dict['pattern']) | |
346 matching_file_list = [f for f in file_list if pattern.search(f)] | |
347 if matching_file_list: | |
348 self._RunHookAction(hook_dict, matching_file_list) | |
349 | 378 |
350 def _RunHookAction(self, hook_dict, matching_file_list): | 379 def _RunHookAction(self, hook_dict, matching_file_list): |
351 """Runs the action from a single hook.""" | 380 """Runs the action from a single hook.""" |
352 logging.info(hook_dict) | 381 logging.info(hook_dict) |
353 logging.info(matching_file_list) | 382 logging.info(matching_file_list) |
354 command = hook_dict['action'][:] | 383 command = hook_dict['action'][:] |
355 if command[0] == 'python': | 384 if command[0] == 'python': |
356 # If the hook specified "python" as the first item, the action is a | 385 # If the hook specified "python" as the first item, the action is a |
357 # Python script. Run it by starting a new copy of the same | 386 # Python script. Run it by starting a new copy of the same |
358 # interpreter. | 387 # interpreter. |
(...skipping 23 matching lines...) Expand all Loading... |
382 def get_custom_deps(self, name, url): | 411 def get_custom_deps(self, name, url): |
383 """Returns a custom deps if applicable.""" | 412 """Returns a custom deps if applicable.""" |
384 if self.parent: | 413 if self.parent: |
385 url = self.parent.get_custom_deps(name, url) | 414 url = self.parent.get_custom_deps(name, url) |
386 # None is a valid return value to disable a dependency. | 415 # None is a valid return value to disable a dependency. |
387 return self.custom_deps.get(name, url) | 416 return self.custom_deps.get(name, url) |
388 | 417 |
389 def __str__(self): | 418 def __str__(self): |
390 out = [] | 419 out = [] |
391 for i in ('name', 'url', 'safesync_url', 'custom_deps', 'custom_vars', | 420 for i in ('name', 'url', 'safesync_url', 'custom_deps', 'custom_vars', |
392 'deps_hooks'): | 421 'deps_hooks', 'file_list'): |
393 # 'deps_file' | 422 # 'deps_file' |
394 if self.__dict__[i]: | 423 if self.__dict__[i]: |
395 out.append('%s: %s' % (i, self.__dict__[i])) | 424 out.append('%s: %s' % (i, self.__dict__[i])) |
396 | 425 |
397 for d in self.dependencies: | 426 for d in self.dependencies: |
398 out.extend([' ' + x for x in str(d).splitlines()]) | 427 out.extend([' ' + x for x in str(d).splitlines()]) |
399 out.append('') | 428 out.append('') |
400 return '\n'.join(out) | 429 return '\n'.join(out) |
401 | 430 |
402 def __repr__(self): | 431 def __repr__(self): |
(...skipping 95 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
498 os.path.join(path, options.config_filename))) | 527 os.path.join(path, options.config_filename))) |
499 return client | 528 return client |
500 | 529 |
501 def SetDefaultConfig(self, solution_name, solution_url, safesync_url): | 530 def SetDefaultConfig(self, solution_name, solution_url, safesync_url): |
502 self.SetConfig(self.DEFAULT_CLIENT_FILE_TEXT % { | 531 self.SetConfig(self.DEFAULT_CLIENT_FILE_TEXT % { |
503 'solution_name': solution_name, | 532 'solution_name': solution_name, |
504 'solution_url': solution_url, | 533 'solution_url': solution_url, |
505 'safesync_url' : safesync_url, | 534 'safesync_url' : safesync_url, |
506 }) | 535 }) |
507 | 536 |
508 def _SaveEntries(self, entries): | 537 def _SaveEntries(self): |
509 """Creates a .gclient_entries file to record the list of unique checkouts. | 538 """Creates a .gclient_entries file to record the list of unique checkouts. |
510 | 539 |
511 The .gclient_entries file lives in the same directory as .gclient. | 540 The .gclient_entries file lives in the same directory as .gclient. |
512 """ | 541 """ |
513 # Sometimes pprint.pformat will use {', sometimes it'll use { ' ... It | 542 # Sometimes pprint.pformat will use {', sometimes it'll use { ' ... It |
514 # makes testing a bit too fun. | 543 # makes testing a bit too fun. |
515 result = pprint.pformat(entries, 2) | 544 result = 'entries = {\n' |
516 if result.startswith('{\''): | 545 for entry in self.tree(False): |
517 result = '{ \'' + result[2:] | 546 result += ' %s: %s,\n' % (pprint.pformat(entry.name), |
518 text = 'entries = \\\n' + result + '\n' | 547 pprint.pformat(entry.parsed_url)) |
| 548 result += '}\n' |
519 file_path = os.path.join(self.root_dir(), self._options.entries_filename) | 549 file_path = os.path.join(self.root_dir(), self._options.entries_filename) |
520 gclient_utils.FileWrite(file_path, text) | 550 logging.info(result) |
| 551 gclient_utils.FileWrite(file_path, result) |
521 | 552 |
522 def _ReadEntries(self): | 553 def _ReadEntries(self): |
523 """Read the .gclient_entries file for the given client. | 554 """Read the .gclient_entries file for the given client. |
524 | 555 |
525 Returns: | 556 Returns: |
526 A sequence of solution names, which will be empty if there is the | 557 A sequence of solution names, which will be empty if there is the |
527 entries file hasn't been created yet. | 558 entries file hasn't been created yet. |
528 """ | 559 """ |
529 scope = {} | 560 scope = {} |
530 filename = os.path.join(self.root_dir(), self._options.entries_filename) | 561 filename = os.path.join(self.root_dir(), self._options.entries_filename) |
(...skipping 40 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
571 def RunOnDeps(self, command, args): | 602 def RunOnDeps(self, command, args): |
572 """Runs a command on each dependency in a client and its dependencies. | 603 """Runs a command on each dependency in a client and its dependencies. |
573 | 604 |
574 Args: | 605 Args: |
575 command: The command to use (e.g., 'status' or 'diff') | 606 command: The command to use (e.g., 'status' or 'diff') |
576 args: list of str - extra arguments to add to the command line. | 607 args: list of str - extra arguments to add to the command line. |
577 """ | 608 """ |
578 if not self.dependencies: | 609 if not self.dependencies: |
579 raise gclient_utils.Error('No solution specified') | 610 raise gclient_utils.Error('No solution specified') |
580 revision_overrides = self._EnforceRevisions() | 611 revision_overrides = self._EnforceRevisions() |
581 | 612 pm = None |
582 # When running runhooks --force, there's no need to consult the SCM. | |
583 # All known hooks are expected to run unconditionally regardless of working | |
584 # copy state, so skip the SCM status check. | |
585 run_scm = not (command == 'runhooks' and self._options.force) | |
586 | |
587 entries = {} | |
588 file_list = [] | |
589 # Run on the base solutions first. | |
590 for solution in self.dependencies: | |
591 name = solution.name | |
592 if name in entries: | |
593 raise gclient_utils.Error("solution %s specified more than once" % name) | |
594 url = solution.url | |
595 entries[name] = url | |
596 if run_scm and url: | |
597 self._options.revision = revision_overrides.get(name) | |
598 scm = gclient_scm.CreateSCM(url, self.root_dir(), name) | |
599 scm.RunCommand(command, self._options, args, file_list) | |
600 file_list = [os.path.join(name, f.strip()) for f in file_list] | |
601 self._options.revision = None | |
602 | |
603 # Process the dependencies next (sort alphanumerically to ensure that | |
604 # containing directories get populated first and for readability) | |
605 deps = self._ParseAllDeps(entries) | |
606 deps_to_process = deps.keys() | |
607 deps_to_process.sort() | |
608 | |
609 # First pass for direct dependencies. | |
610 if command == 'update' and not self._options.verbose: | 613 if command == 'update' and not self._options.verbose: |
611 pm = Progress('Syncing projects', len(deps_to_process)) | 614 pm = Progress('Syncing projects', len(self.tree(False)) + 1) |
612 for d in deps_to_process: | 615 self.RunCommandRecursively(self._options, revision_overrides, |
613 if command == 'update' and not self._options.verbose: | 616 command, args, pm) |
614 pm.update() | 617 if pm: |
615 if type(deps[d]) == str: | |
616 url = deps[d] | |
617 entries[d] = url | |
618 if run_scm: | |
619 self._options.revision = revision_overrides.get(d) | |
620 scm = gclient_scm.CreateSCM(url, self.root_dir(), d) | |
621 scm.RunCommand(command, self._options, args, file_list) | |
622 self._options.revision = None | |
623 elif isinstance(deps[d], self.FileImpl): | |
624 file_dep = deps[d] | |
625 self._options.revision = file_dep.GetRevision() | |
626 if run_scm: | |
627 scm = gclient_scm.CreateSCM(file_dep.GetPath(), self.root_dir(), d) | |
628 scm.RunCommand("updatesingle", self._options, | |
629 args + [file_dep.GetFilename()], file_list) | |
630 | |
631 if command == 'update' and not self._options.verbose: | |
632 pm.end() | 618 pm.end() |
633 | 619 |
634 # Second pass for inherited deps (via the From keyword) | 620 # Once all the dependencies have been processed, it's now safe to run the |
635 for d in deps_to_process: | 621 # hooks. |
636 if isinstance(deps[d], self.FromImpl): | 622 if not self._options.nohooks: |
637 # Getting the URL from the sub_deps file can involve having to resolve | 623 self.RunHooksRecursively(self._options) |
638 # a File() or having to resolve a relative URL. To resolve relative | |
639 # URLs, we need to pass in the orignal sub deps URL. | |
640 sub_deps_base_url = deps[deps[d].module_name] | |
641 sub_deps = Dependency(self, deps[d].module_name, sub_deps_base_url | |
642 ).ParseDepsFile(False) | |
643 url = deps[d].GetUrl(d, sub_deps_base_url, self.root_dir(), sub_deps) | |
644 entries[d] = url | |
645 if run_scm: | |
646 self._options.revision = revision_overrides.get(d) | |
647 scm = gclient_scm.CreateSCM(url, self.root_dir(), d) | |
648 scm.RunCommand(command, self._options, args, file_list) | |
649 self._options.revision = None | |
650 | |
651 # Convert all absolute paths to relative. | |
652 for i in range(len(file_list)): | |
653 # TODO(phajdan.jr): We should know exactly when the paths are absolute. | |
654 # It depends on the command being executed (like runhooks vs sync). | |
655 if not os.path.isabs(file_list[i]): | |
656 continue | |
657 | |
658 prefix = os.path.commonprefix([self.root_dir().lower(), | |
659 file_list[i].lower()]) | |
660 file_list[i] = file_list[i][len(prefix):] | |
661 | |
662 # Strip any leading path separators. | |
663 while file_list[i].startswith('\\') or file_list[i].startswith('/'): | |
664 file_list[i] = file_list[i][1:] | |
665 | |
666 is_using_git = gclient_utils.IsUsingGit(self.root_dir(), entries.keys()) | |
667 self._RunHooks(command, file_list, is_using_git) | |
668 | 624 |
669 if command == 'update': | 625 if command == 'update': |
670 # Notify the user if there is an orphaned entry in their working copy. | 626 # Notify the user if there is an orphaned entry in their working copy. |
671 # Only delete the directory if there are no changes in it, and | 627 # Only delete the directory if there are no changes in it, and |
672 # delete_unversioned_trees is set to true. | 628 # delete_unversioned_trees is set to true. |
673 prev_entries = self._ReadEntries() | 629 entries = [i.name for i in self.tree(False)] |
674 for entry in prev_entries: | 630 for entry, prev_url in self._ReadEntries().iteritems(): |
675 # Fix path separator on Windows. | 631 # Fix path separator on Windows. |
676 entry_fixed = entry.replace('/', os.path.sep) | 632 entry_fixed = entry.replace('/', os.path.sep) |
677 e_dir = os.path.join(self.root_dir(), entry_fixed) | 633 e_dir = os.path.join(self.root_dir(), entry_fixed) |
678 # Use entry and not entry_fixed there. | 634 # Use entry and not entry_fixed there. |
679 if entry not in entries and os.path.exists(e_dir): | 635 if entry not in entries and os.path.exists(e_dir): |
680 modified_files = False | 636 file_list = [] |
681 if isinstance(prev_entries, list): | 637 scm = gclient_scm.CreateSCM(prev_url, self.root_dir(), entry_fixed) |
682 # old .gclient_entries format was list, now dict | 638 scm.status(self._options, [], file_list) |
683 modified_files = gclient_scm.scm.SVN.CaptureStatus(e_dir) | 639 modified_files = file_list != [] |
684 else: | |
685 file_list = [] | |
686 scm = gclient_scm.CreateSCM(prev_entries[entry], self.root_dir(), | |
687 entry_fixed) | |
688 scm.status(self._options, [], file_list) | |
689 modified_files = file_list != [] | |
690 if not self._options.delete_unversioned_trees or modified_files: | 640 if not self._options.delete_unversioned_trees or modified_files: |
691 # There are modified files in this entry. Keep warning until | 641 # There are modified files in this entry. Keep warning until |
692 # removed. | 642 # removed. |
693 print(('\nWARNING: \'%s\' is no longer part of this client. ' | 643 print(('\nWARNING: \'%s\' is no longer part of this client. ' |
694 'It is recommended that you manually remove it.\n') % | 644 'It is recommended that you manually remove it.\n') % |
695 entry_fixed) | 645 entry_fixed) |
696 else: | 646 else: |
697 # Delete the entry | 647 # Delete the entry |
698 print('\n________ deleting \'%s\' in \'%s\'' % ( | 648 print('\n________ deleting \'%s\' in \'%s\'' % ( |
699 entry_fixed, self.root_dir())) | 649 entry_fixed, self.root_dir())) |
700 gclient_utils.RemoveDirectory(e_dir) | 650 gclient_utils.RemoveDirectory(e_dir) |
701 # record the current list of entries for next time | 651 # record the current list of entries for next time |
702 self._SaveEntries(entries) | 652 self._SaveEntries() |
703 return 0 | 653 return 0 |
704 | 654 |
705 def PrintRevInfo(self): | 655 def PrintRevInfo(self): |
706 """Output revision info mapping for the client and its dependencies. | 656 """Output revision info mapping for the client and its dependencies. |
707 | 657 |
708 This allows the capture of an overall "revision" for the source tree that | 658 This allows the capture of an overall "revision" for the source tree that |
709 can be used to reproduce the same tree in the future. It is only useful for | 659 can be used to reproduce the same tree in the future. It is only useful for |
710 "unpinned dependencies", i.e. DEPS/deps references without a svn revision | 660 "unpinned dependencies", i.e. DEPS/deps references without a svn revision |
711 number or a git hash. A git branch name isn't "pinned" since the actual | 661 number or a git hash. A git branch name isn't "pinned" since the actual |
712 commit can change. | 662 commit can change. |
713 | 663 |
714 The --snapshot option allows creating a .gclient file to reproduce the tree. | 664 The --snapshot option allows creating a .gclient file to reproduce the tree. |
715 """ | 665 """ |
716 if not self.dependencies: | 666 if not self.dependencies: |
717 raise gclient_utils.Error('No solution specified') | 667 raise gclient_utils.Error('No solution specified') |
| 668 # Load all the settings. |
| 669 self.RunCommandRecursively(self._options, {}, None, [], None) |
718 | 670 |
719 # Inner helper to generate base url and rev tuple | |
720 def GetURLAndRev(name, original_url): | 671 def GetURLAndRev(name, original_url): |
| 672 """Returns the revision-qualified SCM url.""" |
721 url, _ = gclient_utils.SplitUrlRevision(original_url) | 673 url, _ = gclient_utils.SplitUrlRevision(original_url) |
722 scm = gclient_scm.CreateSCM(original_url, self.root_dir(), name) | 674 scm = gclient_scm.CreateSCM(original_url, self.root_dir(), name) |
723 return (url, scm.revinfo(self._options, [], None)) | 675 if not os.path.isdir(scm.checkout_path): |
| 676 return None |
| 677 return '%s@%s' % (url, scm.revinfo(self._options, [], None)) |
724 | 678 |
725 # text of the snapshot gclient file | |
726 new_gclient = "" | |
727 # Dictionary of { path : SCM url } to ensure no duplicate solutions | |
728 solution_names = {} | |
729 entries = {} | |
730 # Run on the base solutions first. | |
731 for solution in self.dependencies: | |
732 # Dictionary of { path : SCM url } to describe the gclient checkout | |
733 name = solution.name | |
734 if name in solution_names: | |
735 raise gclient_utils.Error("solution %s specified more than once" % name) | |
736 (url, rev) = GetURLAndRev(name, solution.url) | |
737 entries[name] = "%s@%s" % (url, rev) | |
738 solution_names[name] = "%s@%s" % (url, rev) | |
739 | |
740 # Process the dependencies next (sort alphanumerically to ensure that | |
741 # containing directories get populated first and for readability) | |
742 deps = self._ParseAllDeps(entries) | |
743 deps_to_process = deps.keys() | |
744 deps_to_process.sort() | |
745 | |
746 # First pass for direct dependencies. | |
747 for d in deps_to_process: | |
748 if type(deps[d]) == str: | |
749 (url, rev) = GetURLAndRev(d, deps[d]) | |
750 entries[d] = "%s@%s" % (url, rev) | |
751 | |
752 # Second pass for inherited deps (via the From keyword) | |
753 for d in deps_to_process: | |
754 if isinstance(deps[d], self.FromImpl): | |
755 deps_parent_url = entries[deps[d].module_name] | |
756 if deps_parent_url.find("@") < 0: | |
757 raise gclient_utils.Error("From %s missing revisioned url" % | |
758 deps[d].module_name) | |
759 sub_deps_base_url = deps[deps[d].module_name] | |
760 sub_deps = Dependency(self, deps[d].module_name, sub_deps_base_url | |
761 ).ParseDepsFile(False) | |
762 url = deps[d].GetUrl(d, sub_deps_base_url, self.root_dir(), sub_deps) | |
763 (url, rev) = GetURLAndRev(d, url) | |
764 entries[d] = "%s@%s" % (url, rev) | |
765 | |
766 # Build the snapshot configuration string | |
767 if self._options.snapshot: | 679 if self._options.snapshot: |
768 url = entries.pop(name) | 680 new_gclient = '' |
769 custom_deps = ''.join([' \"%s\": \"%s\",\n' % (x, entries[x]) | 681 # First level at .gclient |
770 for x in sorted(entries.keys())]) | 682 for d in self.dependencies: |
771 | 683 entries = {} |
772 new_gclient += self.DEFAULT_SNAPSHOT_SOLUTION_TEXT % { | 684 def GrabDeps(sol): |
773 'solution_name': name, | 685 """Recursively grab dependencies.""" |
774 'solution_url': url, | 686 for i in sol.dependencies: |
775 'safesync_url' : '', | 687 entries[i.name] = GetURLAndRev(i.name, i.parsed_url) |
776 'solution_deps': custom_deps, | 688 GrabDeps(i) |
777 } | 689 GrabDeps(d) |
| 690 custom_deps = [] |
| 691 for k in sorted(entries.keys()): |
| 692 if entries[k]: |
| 693 # Quotes aren't escaped... |
| 694 custom_deps.append(' \"%s\": \'%s\',\n' % (k, entries[k])) |
| 695 else: |
| 696 custom_deps.append(' \"%s\": None,\n' % k) |
| 697 new_gclient += self.DEFAULT_SNAPSHOT_SOLUTION_TEXT % { |
| 698 'solution_name': d.name, |
| 699 'solution_url': d.url, |
| 700 'safesync_url' : d.safesync_url or '', |
| 701 'solution_deps': ''.join(custom_deps), |
| 702 } |
| 703 # Print the snapshot configuration file |
| 704 print(self.DEFAULT_SNAPSHOT_FILE_TEXT % {'solution_list': new_gclient}) |
778 else: | 705 else: |
779 print(';\n'.join(['%s: %s' % (x, entries[x]) | 706 entries = sorted(self.tree(False), key=lambda i: i.name) |
780 for x in sorted(entries.keys())])) | 707 for entry in entries: |
781 | 708 revision = GetURLAndRev(entry.name, entry.parsed_url) |
782 # Print the snapshot configuration file | 709 line = '%s: %s' % (entry.name, revision) |
783 if self._options.snapshot: | 710 if not entry is entries[-1]: |
784 config = self.DEFAULT_SNAPSHOT_FILE_TEXT % {'solution_list': new_gclient} | 711 line += ';' |
785 snapclient = GClient(self.root_dir(), self._options) | 712 print line |
786 snapclient.SetConfig(config) | |
787 print(snapclient.config_content) | |
788 | 713 |
789 def ParseDepsFile(self, direct_reference): | 714 def ParseDepsFile(self, direct_reference): |
790 """No DEPS to parse for a .gclient file.""" | 715 """No DEPS to parse for a .gclient file.""" |
791 self.direct_reference = direct_reference | 716 self.direct_reference = direct_reference |
792 self.deps_parsed = True | 717 self.deps_parsed = True |
793 | 718 |
794 def root_dir(self): | 719 def root_dir(self): |
795 """Root directory of gclient checkout.""" | 720 """Root directory of gclient checkout.""" |
796 return self._root_dir | 721 return self._root_dir |
797 | 722 |
(...skipping 300 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1098 'unpinned dependencies', i.e. DEPS/deps references without a svn revision | 1023 'unpinned dependencies', i.e. DEPS/deps references without a svn revision |
1099 number or a git hash. A git branch name isn't 'pinned' since the actual | 1024 number or a git hash. A git branch name isn't 'pinned' since the actual |
1100 commit can change. | 1025 commit can change. |
1101 """ | 1026 """ |
1102 parser.add_option('--deps', dest='deps_os', metavar='OS_LIST', | 1027 parser.add_option('--deps', dest='deps_os', metavar='OS_LIST', |
1103 help='override deps for the specified (comma-separated) ' | 1028 help='override deps for the specified (comma-separated) ' |
1104 'platform(s); \'all\' will process all deps_os ' | 1029 'platform(s); \'all\' will process all deps_os ' |
1105 'references') | 1030 'references') |
1106 parser.add_option('-s', '--snapshot', action='store_true', | 1031 parser.add_option('-s', '--snapshot', action='store_true', |
1107 help='creates a snapshot .gclient file of the current ' | 1032 help='creates a snapshot .gclient file of the current ' |
1108 'version of all repositories to reproduce the tree, ' | 1033 'version of all repositories to reproduce the tree') |
1109 'implies -a') | |
1110 (options, args) = parser.parse_args(args) | 1034 (options, args) = parser.parse_args(args) |
1111 client = GClient.LoadCurrentConfig(options) | 1035 client = GClient.LoadCurrentConfig(options) |
1112 if not client: | 1036 if not client: |
1113 raise gclient_utils.Error('client not configured; see \'gclient config\'') | 1037 raise gclient_utils.Error('client not configured; see \'gclient config\'') |
1114 client.PrintRevInfo() | 1038 client.PrintRevInfo() |
1115 return 0 | 1039 return 0 |
1116 | 1040 |
1117 | 1041 |
1118 def Command(name): | 1042 def Command(name): |
1119 return getattr(sys.modules[__name__], 'CMD' + name, None) | 1043 return getattr(sys.modules[__name__], 'CMD' + name, None) |
(...skipping 71 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1191 return CMDhelp(parser, argv) | 1115 return CMDhelp(parser, argv) |
1192 except gclient_utils.Error, e: | 1116 except gclient_utils.Error, e: |
1193 print >> sys.stderr, 'Error: %s' % str(e) | 1117 print >> sys.stderr, 'Error: %s' % str(e) |
1194 return 1 | 1118 return 1 |
1195 | 1119 |
1196 | 1120 |
1197 if '__main__' == __name__: | 1121 if '__main__' == __name__: |
1198 sys.exit(Main(sys.argv[1:])) | 1122 sys.exit(Main(sys.argv[1:])) |
1199 | 1123 |
1200 # vim: ts=2:sw=2:tw=80:et: | 1124 # vim: ts=2:sw=2:tw=80:et: |
OLD | NEW |