Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(89)

Side by Side Diff: gclient.py

Issue 2867047: Bring some OOP and sanity to gclient.py. (Closed)
Patch Set: Reuse old code Created 10 years, 5 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | gclient_utils.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/python 1 #!/usr/bin/python
2 # Copyright (c) 2010 The Chromium Authors. All rights reserved. 2 # Copyright (c) 2010 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 5
6 """Meta checkout manager supporting both Subversion and GIT. 6 """Meta checkout manager supporting both Subversion and GIT.
7 7
8 Files 8 Files
9 .gclient : Current client configuration, written by 'config' command. 9 .gclient : Current client configuration, written by 'config' command.
10 Format is a Python script defining 'solutions', a list whose 10 Format is a Python script defining 'solutions', a list whose
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after
42 it will be removed from the list and the list will be extended 42 it will be removed from the list and the list will be extended
43 by the list of matching files. 43 by the list of matching files.
44 44
45 Example: 45 Example:
46 hooks = [ 46 hooks = [
47 { "pattern": "\\.(gif|jpe?g|pr0n|png)$", 47 { "pattern": "\\.(gif|jpe?g|pr0n|png)$",
48 "action": ["python", "image_indexer.py", "--all"]}, 48 "action": ["python", "image_indexer.py", "--all"]},
49 ] 49 ]
50 """ 50 """
51 51
52 __version__ = "0.4.1" 52 __version__ = "0.5"
53 53
54 import errno
55 import logging 54 import logging
56 import optparse 55 import optparse
57 import os 56 import os
58 import pprint 57 import pprint
59 import re 58 import re
60 import subprocess 59 import subprocess
61 import sys 60 import sys
62 import urlparse 61 import urlparse
63 import urllib 62 import urllib
64 63
(...skipping 25 matching lines...) Expand all
90 89
91 sub_target_name is an optional parameter if the module name in the other 90 sub_target_name is an optional parameter if the module name in the other
92 DEPS file is different. E.g., you might want to map src/net to net.""" 91 DEPS file is different. E.g., you might want to map src/net to net."""
93 self.module_name = module_name 92 self.module_name = module_name
94 self.sub_target_name = sub_target_name 93 self.sub_target_name = sub_target_name
95 94
96 def __str__(self): 95 def __str__(self):
97 return 'From(%s, %s)' % (repr(self.module_name), 96 return 'From(%s, %s)' % (repr(self.module_name),
98 repr(self.sub_target_name)) 97 repr(self.sub_target_name))
99 98
100 def GetUrl(self, target_name, sub_deps_base_url, root_dir, sub_deps):
101 """Resolve the URL for this From entry."""
102 sub_deps_target_name = target_name
103 if self.sub_target_name:
104 sub_deps_target_name = self.sub_target_name
105 url = sub_deps[sub_deps_target_name]
106 if url.startswith('/'):
107 # If it's a relative URL, we need to resolve the URL relative to the
108 # sub deps base URL.
109 if not isinstance(sub_deps_base_url, basestring):
110 sub_deps_base_url = sub_deps_base_url.GetPath()
111 scm = gclient_scm.CreateSCM(sub_deps_base_url, root_dir,
112 None)
113 url = scm.FullUrlForRelativeUrl(url)
114 return url
115
116 class FileImpl(object): 99 class FileImpl(object):
117 """Used to implement the File('') syntax which lets you sync a single file 100 """Used to implement the File('') syntax which lets you sync a single file
118 from an SVN repo.""" 101 from an SVN repo."""
119 102
120 def __init__(self, file_location): 103 def __init__(self, file_location):
121 self.file_location = file_location 104 self.file_location = file_location
122 105
123 def __str__(self): 106 def __str__(self):
124 return 'File("%s")' % self.file_location 107 return 'File("%s")' % self.file_location
125 108
(...skipping 27 matching lines...) Expand all
153 class Dependency(GClientKeywords): 136 class Dependency(GClientKeywords):
154 """Object that represents a dependency checkout.""" 137 """Object that represents a dependency checkout."""
155 DEPS_FILE = 'DEPS' 138 DEPS_FILE = 'DEPS'
156 139
157 def __init__(self, parent, name, url, safesync_url=None, custom_deps=None, 140 def __init__(self, parent, name, url, safesync_url=None, custom_deps=None,
158 custom_vars=None, deps_file=None): 141 custom_vars=None, deps_file=None):
159 GClientKeywords.__init__(self) 142 GClientKeywords.__init__(self)
160 self.parent = parent 143 self.parent = parent
161 self.name = name 144 self.name = name
162 self.url = url 145 self.url = url
146 self.parsed_url = None
163 # These 2 are only set in .gclient and not in DEPS files. 147 # These 2 are only set in .gclient and not in DEPS files.
164 self.safesync_url = safesync_url 148 self.safesync_url = safesync_url
165 self.custom_vars = custom_vars or {} 149 self.custom_vars = custom_vars or {}
166 self.custom_deps = custom_deps or {} 150 self.custom_deps = custom_deps or {}
167 self.deps_hooks = [] 151 self.deps_hooks = []
168 self.dependencies = [] 152 self.dependencies = []
169 self.deps_file = deps_file or self.DEPS_FILE 153 self.deps_file = deps_file or self.DEPS_FILE
154 # A cache of the files affected by the current operation, necessary for
155 # hooks.
156 self.file_list = []
170 self.deps_parsed = False 157 self.deps_parsed = False
171 self.direct_reference = False 158 self.direct_reference = False
172 159
173 # Sanity checks 160 # Sanity checks
174 if not self.name and self.parent: 161 if not self.name and self.parent:
175 raise gclient_utils.Error('Dependency without name') 162 raise gclient_utils.Error('Dependency without name')
163 if self.name in [d.name for d in self.tree(False)]:
164 raise gclient_utils.Error('Dependency %s specified more than once' %
165 self.name)
176 if not isinstance(self.url, 166 if not isinstance(self.url,
177 (basestring, self.FromImpl, self.FileImpl, None.__class__)): 167 (basestring, self.FromImpl, self.FileImpl, None.__class__)):
178 raise gclient_utils.Error('dependency url must be either a string, None, ' 168 raise gclient_utils.Error('dependency url must be either a string, None, '
179 'File() or From() instead of %s' % 169 'File() or From() instead of %s' %
180 self.url.__class__.__name__) 170 self.url.__class__.__name__)
181 if '/' in self.deps_file or '\\' in self.deps_file: 171 if '/' in self.deps_file or '\\' in self.deps_file:
182 raise gclient_utils.Error('deps_file name must not be a path, just a ' 172 raise gclient_utils.Error('deps_file name must not be a path, just a '
183 'filename. %s' % self.deps_file) 173 'filename. %s' % self.deps_file)
184 174
175 def LateOverride(self, url):
176 overriden_url = self.get_custom_deps(self.name, url)
177 if overriden_url != url:
178 self.parsed_url = overriden_url
179 logging.debug('%s, %s was overriden to %s' % (self.name, url,
180 self.parsed_url))
181 elif isinstance(url, self.FromImpl):
182 ref = [dep for dep in self.tree(True) if url.module_name == dep.name]
183 if not len(ref) == 1:
184 raise Exception('Failed to find one reference to %s. %s' % (
185 url.module_name, ref))
186 ref = ref[0]
187 sub_target = url.sub_target_name or url
188 # Make sure the referenced dependency DEPS file is loaded and file the
189 # inner referenced dependency.
190 ref.ParseDepsFile(False)
191 found_dep = None
192 for d in ref.dependencies:
193 if d.name == sub_target:
194 found_dep = d
195 break
196 if not found_dep:
197 raise Exception('Couldn\'t find %s in %s, referenced by %s' % (
198 sub_target, ref.name, self.name))
199 # Call LateOverride() again.
200 self.parsed_url = found_dep.LateOverride(found_dep.url)
201 logging.debug('%s, %s to %s' % (self.name, url, self.parsed_url))
202 elif isinstance(url, basestring):
203 parsed_url = urlparse.urlparse(url)
204 if not parsed_url[0]:
205 # A relative url. Fetch the real base.
206 path = parsed_url[2]
207 if not path.startswith('/'):
208 raise gclient_utils.Error(
209 'relative DEPS entry \'%s\' must begin with a slash' % url)
210 # Create a scm just to query the full url.
211 scm = gclient_scm.CreateSCM(self.parent.parsed_url, self.root_dir(),
212 None)
213 self.parsed_url = scm.FullUrlForRelativeUrl(url)
214 else:
215 self.parsed_url = url
216 logging.debug('%s, %s -> %s' % (self.name, url, self.parsed_url))
217 elif isinstance(url, self.FileImpl):
218 self.parsed_url = url
219 logging.debug('%s, %s -> %s (File)' % (self.name, url, self.parsed_url))
220 return self.parsed_url
221
185 def ParseDepsFile(self, direct_reference): 222 def ParseDepsFile(self, direct_reference):
186 """Parses the DEPS file for this dependency.""" 223 """Parses the DEPS file for this dependency."""
187 if direct_reference: 224 if direct_reference:
188 # Maybe it was referenced earlier by a From() keyword but it's now 225 # Maybe it was referenced earlier by a From() keyword but it's now
189 # directly referenced. 226 # directly referenced.
190 self.direct_reference = direct_reference 227 self.direct_reference = direct_reference
228 if self.deps_parsed:
229 return
191 self.deps_parsed = True 230 self.deps_parsed = True
192 filepath = os.path.join(self.root_dir(), self.name, self.deps_file) 231 filepath = os.path.join(self.root_dir(), self.name, self.deps_file)
193 if not os.path.isfile(filepath): 232 if not os.path.isfile(filepath):
194 return {} 233 return
195 deps_content = gclient_utils.FileRead(filepath) 234 deps_content = gclient_utils.FileRead(filepath)
196 235
197 # Eval the content. 236 # Eval the content.
198 # One thing is unintuitive, vars= {} must happen before Var() use. 237 # One thing is unintuitive, vars= {} must happen before Var() use.
199 local_scope = {} 238 local_scope = {}
200 var = self.VarImpl(self.custom_vars, local_scope) 239 var = self.VarImpl(self.custom_vars, local_scope)
201 global_scope = { 240 global_scope = {
202 'File': self.FileImpl, 241 'File': self.FileImpl,
203 'From': self.FromImpl, 242 'From': self.FromImpl,
204 'Var': var.Lookup, 243 'Var': var.Lookup,
(...skipping 30 matching lines...) Expand all
235 # the dictionary using paths relative to the directory containing 274 # the dictionary using paths relative to the directory containing
236 # the DEPS file. 275 # the DEPS file.
237 use_relative_paths = local_scope.get('use_relative_paths', False) 276 use_relative_paths = local_scope.get('use_relative_paths', False)
238 if use_relative_paths: 277 if use_relative_paths:
239 rel_deps = {} 278 rel_deps = {}
240 for d, url in deps.items(): 279 for d, url in deps.items():
241 # normpath is required to allow DEPS to use .. in their 280 # normpath is required to allow DEPS to use .. in their
242 # dependency local path. 281 # dependency local path.
243 rel_deps[os.path.normpath(os.path.join(self.name, d))] = url 282 rel_deps[os.path.normpath(os.path.join(self.name, d))] = url
244 deps = rel_deps 283 deps = rel_deps
245 # TODO(maruel): Add these dependencies into self.dependencies.
246 return deps
247 284
248 def _ParseAllDeps(self, solution_urls): 285 # Convert the deps into real Dependency.
249 """Parse the complete list of dependencies for the client. 286 for name, url in deps.iteritems():
287 if name in [s.name for s in self.dependencies]:
288 raise
289 self.dependencies.append(Dependency(self, name, url))
290 # Sort by name.
291 self.dependencies.sort(key=lambda x: x.name)
292 logging.info('Loaded: %s' % str(self))
250 293
251 Args: 294 def RunCommandRecursively(self, options, revision_overrides,
252 solution_urls: A dict mapping module names (as relative paths) to URLs 295 command, args, pm):
253 corresponding to the solutions specified by the client. This parameter 296 """Runs 'command' before parsing the DEPS in case it's a initial checkout
254 is passed as an optimization. 297 or a revert."""
298 assert self.file_list == []
299 # When running runhooks, there's no need to consult the SCM.
300 # All known hooks are expected to run unconditionally regardless of working
301 # copy state, so skip the SCM status check.
302 run_scm = command not in ('runhooks', None)
303 self.LateOverride(self.url)
304 if run_scm and self.parsed_url:
305 if isinstance(self.parsed_url, self.FileImpl):
306 # Special support for single-file checkout.
307 options.revision = self.parsed_url.GetRevision()
308 scm = gclient_scm.CreateSCM(self.parsed_url.GetPath(), self.root_dir(),
309 self.name)
310 scm.RunCommand("updatesingle", options,
311 args + [self.parsed_url.GetFilename()], self.file_list)
312 else:
313 options.revision = revision_overrides.get(self.name)
314 scm = gclient_scm.CreateSCM(self.parsed_url, self.root_dir(), self.name)
315 scm.RunCommand(command, options, args, self.file_list)
316 self.file_list = [os.path.join(self.name, f.strip())
317 for f in self.file_list]
318 options.revision = None
319 if pm:
320 # The + 1 comes from the fact that .gclient is considered a step in
321 # itself, .i.e. this code is called one time for the .gclient. This is not
322 # conceptually correct but it simplifies code.
323 pm._total = len(self.tree(False)) + 1
324 pm.update()
325 if self.recursion_limit():
326 # Then we can parse the DEPS file.
327 self.ParseDepsFile(True)
328 if pm:
329 pm._total = len(self.tree(False)) + 1
330 pm.update(0)
331 # Parse the dependencies of this dependency.
332 for s in self.dependencies:
333 # TODO(maruel): All these can run concurrently! No need for threads,
334 # just buffer stdout&stderr on pipes and flush as they complete.
335 # Watch out for stdin.
336 s.RunCommandRecursively(options, revision_overrides, command, args, pm)
255 337
256 Returns: 338 def RunHooksRecursively(self, options):
257 A dict mapping module names (as relative paths) to URLs corresponding 339 """Evaluates all hooks, running actions as needed. RunCommandRecursively()
258 to the entire set of dependencies to checkout for the given client. 340 must have been called before to load the DEPS."""
341 # If "--force" was specified, run all hooks regardless of what files have
342 # changed.
343 if self.deps_hooks:
344 # TODO(maruel): If the user is using git or git-svn, then we don't know
345 # what files have changed so we always run all hooks. It'd be nice to fix
346 # that.
347 if (options.force or
348 gclient_scm.GetScmName(self.parsed_url) in ('git', None) or
349 os.path.isdir(os.path.join(self.root_dir(), self.name, '.git'))):
350 for hook_dict in self.deps_hooks:
351 self._RunHookAction(hook_dict, [])
352 else:
353 # TODO(phajdan.jr): We should know exactly when the paths are absolute.
354 # Convert all absolute paths to relative.
355 for i in range(len(self.file_list)):
356 # It depends on the command being executed (like runhooks vs sync).
357 if not os.path.isabs(self.file_list[i]):
358 continue
259 359
260 Raises: 360 prefix = os.path.commonprefix([self.root_dir().lower(),
261 Error: If a dependency conflicts with another dependency or of a solution. 361 self.file_list[i].lower()])
262 """ 362 self.file_list[i] = self.file_list[i][len(prefix):]
263 deps = {}
264 for solution in self.dependencies:
265 solution_deps = solution.ParseDepsFile(True)
266 363
267 # If a line is in custom_deps, but not in the solution, we want to append 364 # Strip any leading path separators.
268 # this line to the solution. 365 while file_list[i].startswith('\\') or file_list[i].startswith('/'):
269 for d in solution.custom_deps: 366 self.file_list[i] = self.file_list[i][1:]
270 if d not in solution_deps:
271 solution_deps[d] = solution.custom_deps[d]
272 367
273 for d in solution_deps: 368 # Run hooks on the basis of whether the files from the gclient operation
274 if d in solution.custom_deps: 369 # match each hook's pattern.
275 # Dependency is overriden. 370 for hook_dict in self.deps_hooks:
276 url = solution.custom_deps[d] 371 pattern = re.compile(hook_dict['pattern'])
277 if url is None: 372 matching_file_list = [f for f in self.file_list if pattern.search(f)]
278 continue 373 if matching_file_list:
279 else: 374 self._RunHookAction(hook_dict, matching_file_list)
280 url = solution_deps[d] 375 if self.recursion_limit():
281 # if we have a From reference dependent on another solution, then 376 for s in self.dependencies:
282 # just skip the From reference. When we pull deps for the solution, 377 s.RunHooksRecursively(options)
283 # we will take care of this dependency.
284 #
285 # If multiple solutions all have the same From reference, then we
286 # should only add one to our list of dependencies.
287 if isinstance(url, self.FromImpl):
288 if url.module_name in solution_urls:
289 # Already parsed.
290 continue
291 if d in deps and type(deps[d]) != str:
292 if url.module_name == deps[d].module_name:
293 continue
294 elif isinstance(url, str):
295 parsed_url = urlparse.urlparse(url)
296 scheme = parsed_url[0]
297 if not scheme:
298 # A relative url. Fetch the real base.
299 path = parsed_url[2]
300 if path[0] != "/":
301 raise gclient_utils.Error(
302 "relative DEPS entry \"%s\" must begin with a slash" % d)
303 # Create a scm just to query the full url.
304 scm = gclient_scm.CreateSCM(solution.url, self.root_dir(),
305 None)
306 url = scm.FullUrlForRelativeUrl(url)
307 if d in deps and deps[d] != url:
308 raise gclient_utils.Error(
309 "Solutions have conflicting versions of dependency \"%s\"" % d)
310 if d in solution_urls and solution_urls[d] != url:
311 raise gclient_utils.Error(
312 "Dependency \"%s\" conflicts with specified solution" % d)
313 # Grab the dependency.
314 deps[d] = url
315 return deps
316
317 def _RunHooks(self, command, file_list, is_using_git):
318 """Evaluates all hooks, running actions as needed.
319 """
320 # Hooks only run for these command types.
321 if not command in ('update', 'revert', 'runhooks'):
322 return
323
324 # Hooks only run when --nohooks is not specified
325 if self._options.nohooks:
326 return
327
328 # Get any hooks from the .gclient file.
329 hooks = self.deps_hooks[:]
330 # Add any hooks found in DEPS files.
331 for d in self.dependencies:
332 hooks.extend(d.deps_hooks)
333
334 # If "--force" was specified, run all hooks regardless of what files have
335 # changed. If the user is using git, then we don't know what files have
336 # changed so we always run all hooks.
337 if self._options.force or is_using_git:
338 for hook_dict in hooks:
339 self._RunHookAction(hook_dict, [])
340 return
341
342 # Run hooks on the basis of whether the files from the gclient operation
343 # match each hook's pattern.
344 for hook_dict in hooks:
345 pattern = re.compile(hook_dict['pattern'])
346 matching_file_list = [f for f in file_list if pattern.search(f)]
347 if matching_file_list:
348 self._RunHookAction(hook_dict, matching_file_list)
349 378
350 def _RunHookAction(self, hook_dict, matching_file_list): 379 def _RunHookAction(self, hook_dict, matching_file_list):
351 """Runs the action from a single hook.""" 380 """Runs the action from a single hook."""
352 logging.info(hook_dict) 381 logging.info(hook_dict)
353 logging.info(matching_file_list) 382 logging.info(matching_file_list)
354 command = hook_dict['action'][:] 383 command = hook_dict['action'][:]
355 if command[0] == 'python': 384 if command[0] == 'python':
356 # If the hook specified "python" as the first item, the action is a 385 # If the hook specified "python" as the first item, the action is a
357 # Python script. Run it by starting a new copy of the same 386 # Python script. Run it by starting a new copy of the same
358 # interpreter. 387 # interpreter.
(...skipping 23 matching lines...) Expand all
382 def get_custom_deps(self, name, url): 411 def get_custom_deps(self, name, url):
383 """Returns a custom deps if applicable.""" 412 """Returns a custom deps if applicable."""
384 if self.parent: 413 if self.parent:
385 url = self.parent.get_custom_deps(name, url) 414 url = self.parent.get_custom_deps(name, url)
386 # None is a valid return value to disable a dependency. 415 # None is a valid return value to disable a dependency.
387 return self.custom_deps.get(name, url) 416 return self.custom_deps.get(name, url)
388 417
389 def __str__(self): 418 def __str__(self):
390 out = [] 419 out = []
391 for i in ('name', 'url', 'safesync_url', 'custom_deps', 'custom_vars', 420 for i in ('name', 'url', 'safesync_url', 'custom_deps', 'custom_vars',
392 'deps_hooks'): 421 'deps_hooks', 'file_list'):
393 # 'deps_file' 422 # 'deps_file'
394 if self.__dict__[i]: 423 if self.__dict__[i]:
395 out.append('%s: %s' % (i, self.__dict__[i])) 424 out.append('%s: %s' % (i, self.__dict__[i]))
396 425
397 for d in self.dependencies: 426 for d in self.dependencies:
398 out.extend([' ' + x for x in str(d).splitlines()]) 427 out.extend([' ' + x for x in str(d).splitlines()])
399 out.append('') 428 out.append('')
400 return '\n'.join(out) 429 return '\n'.join(out)
401 430
402 def __repr__(self): 431 def __repr__(self):
(...skipping 95 matching lines...) Expand 10 before | Expand all | Expand 10 after
498 os.path.join(path, options.config_filename))) 527 os.path.join(path, options.config_filename)))
499 return client 528 return client
500 529
501 def SetDefaultConfig(self, solution_name, solution_url, safesync_url): 530 def SetDefaultConfig(self, solution_name, solution_url, safesync_url):
502 self.SetConfig(self.DEFAULT_CLIENT_FILE_TEXT % { 531 self.SetConfig(self.DEFAULT_CLIENT_FILE_TEXT % {
503 'solution_name': solution_name, 532 'solution_name': solution_name,
504 'solution_url': solution_url, 533 'solution_url': solution_url,
505 'safesync_url' : safesync_url, 534 'safesync_url' : safesync_url,
506 }) 535 })
507 536
508 def _SaveEntries(self, entries): 537 def _SaveEntries(self):
509 """Creates a .gclient_entries file to record the list of unique checkouts. 538 """Creates a .gclient_entries file to record the list of unique checkouts.
510 539
511 The .gclient_entries file lives in the same directory as .gclient. 540 The .gclient_entries file lives in the same directory as .gclient.
512 """ 541 """
513 # Sometimes pprint.pformat will use {', sometimes it'll use { ' ... It 542 # Sometimes pprint.pformat will use {', sometimes it'll use { ' ... It
514 # makes testing a bit too fun. 543 # makes testing a bit too fun.
515 result = pprint.pformat(entries, 2) 544 result = 'entries = {\n'
516 if result.startswith('{\''): 545 for entry in self.tree(False):
517 result = '{ \'' + result[2:] 546 result += ' %s: %s,\n' % (pprint.pformat(entry.name),
518 text = 'entries = \\\n' + result + '\n' 547 pprint.pformat(entry.parsed_url))
548 result += '}\n'
519 file_path = os.path.join(self.root_dir(), self._options.entries_filename) 549 file_path = os.path.join(self.root_dir(), self._options.entries_filename)
520 gclient_utils.FileWrite(file_path, text) 550 logging.info(result)
551 gclient_utils.FileWrite(file_path, result)
521 552
522 def _ReadEntries(self): 553 def _ReadEntries(self):
523 """Read the .gclient_entries file for the given client. 554 """Read the .gclient_entries file for the given client.
524 555
525 Returns: 556 Returns:
526 A sequence of solution names, which will be empty if there is the 557 A sequence of solution names, which will be empty if there is the
527 entries file hasn't been created yet. 558 entries file hasn't been created yet.
528 """ 559 """
529 scope = {} 560 scope = {}
530 filename = os.path.join(self.root_dir(), self._options.entries_filename) 561 filename = os.path.join(self.root_dir(), self._options.entries_filename)
(...skipping 40 matching lines...) Expand 10 before | Expand all | Expand 10 after
571 def RunOnDeps(self, command, args): 602 def RunOnDeps(self, command, args):
572 """Runs a command on each dependency in a client and its dependencies. 603 """Runs a command on each dependency in a client and its dependencies.
573 604
574 Args: 605 Args:
575 command: The command to use (e.g., 'status' or 'diff') 606 command: The command to use (e.g., 'status' or 'diff')
576 args: list of str - extra arguments to add to the command line. 607 args: list of str - extra arguments to add to the command line.
577 """ 608 """
578 if not self.dependencies: 609 if not self.dependencies:
579 raise gclient_utils.Error('No solution specified') 610 raise gclient_utils.Error('No solution specified')
580 revision_overrides = self._EnforceRevisions() 611 revision_overrides = self._EnforceRevisions()
581 612 pm = None
582 # When running runhooks --force, there's no need to consult the SCM.
583 # All known hooks are expected to run unconditionally regardless of working
584 # copy state, so skip the SCM status check.
585 run_scm = not (command == 'runhooks' and self._options.force)
586
587 entries = {}
588 file_list = []
589 # Run on the base solutions first.
590 for solution in self.dependencies:
591 name = solution.name
592 if name in entries:
593 raise gclient_utils.Error("solution %s specified more than once" % name)
594 url = solution.url
595 entries[name] = url
596 if run_scm and url:
597 self._options.revision = revision_overrides.get(name)
598 scm = gclient_scm.CreateSCM(url, self.root_dir(), name)
599 scm.RunCommand(command, self._options, args, file_list)
600 file_list = [os.path.join(name, f.strip()) for f in file_list]
601 self._options.revision = None
602
603 # Process the dependencies next (sort alphanumerically to ensure that
604 # containing directories get populated first and for readability)
605 deps = self._ParseAllDeps(entries)
606 deps_to_process = deps.keys()
607 deps_to_process.sort()
608
609 # First pass for direct dependencies.
610 if command == 'update' and not self._options.verbose: 613 if command == 'update' and not self._options.verbose:
611 pm = Progress('Syncing projects', len(deps_to_process)) 614 pm = Progress('Syncing projects', len(self.tree(False)) + 1)
612 for d in deps_to_process: 615 self.RunCommandRecursively(self._options, revision_overrides,
613 if command == 'update' and not self._options.verbose: 616 command, args, pm)
614 pm.update() 617 if pm:
615 if type(deps[d]) == str:
616 url = deps[d]
617 entries[d] = url
618 if run_scm:
619 self._options.revision = revision_overrides.get(d)
620 scm = gclient_scm.CreateSCM(url, self.root_dir(), d)
621 scm.RunCommand(command, self._options, args, file_list)
622 self._options.revision = None
623 elif isinstance(deps[d], self.FileImpl):
624 file_dep = deps[d]
625 self._options.revision = file_dep.GetRevision()
626 if run_scm:
627 scm = gclient_scm.CreateSCM(file_dep.GetPath(), self.root_dir(), d)
628 scm.RunCommand("updatesingle", self._options,
629 args + [file_dep.GetFilename()], file_list)
630
631 if command == 'update' and not self._options.verbose:
632 pm.end() 618 pm.end()
633 619
634 # Second pass for inherited deps (via the From keyword) 620 # Once all the dependencies have been processed, it's now safe to run the
635 for d in deps_to_process: 621 # hooks.
636 if isinstance(deps[d], self.FromImpl): 622 if not self._options.nohooks:
637 # Getting the URL from the sub_deps file can involve having to resolve 623 self.RunHooksRecursively(self._options)
638 # a File() or having to resolve a relative URL. To resolve relative
639 # URLs, we need to pass in the orignal sub deps URL.
640 sub_deps_base_url = deps[deps[d].module_name]
641 sub_deps = Dependency(self, deps[d].module_name, sub_deps_base_url
642 ).ParseDepsFile(False)
643 url = deps[d].GetUrl(d, sub_deps_base_url, self.root_dir(), sub_deps)
644 entries[d] = url
645 if run_scm:
646 self._options.revision = revision_overrides.get(d)
647 scm = gclient_scm.CreateSCM(url, self.root_dir(), d)
648 scm.RunCommand(command, self._options, args, file_list)
649 self._options.revision = None
650
651 # Convert all absolute paths to relative.
652 for i in range(len(file_list)):
653 # TODO(phajdan.jr): We should know exactly when the paths are absolute.
654 # It depends on the command being executed (like runhooks vs sync).
655 if not os.path.isabs(file_list[i]):
656 continue
657
658 prefix = os.path.commonprefix([self.root_dir().lower(),
659 file_list[i].lower()])
660 file_list[i] = file_list[i][len(prefix):]
661
662 # Strip any leading path separators.
663 while file_list[i].startswith('\\') or file_list[i].startswith('/'):
664 file_list[i] = file_list[i][1:]
665
666 is_using_git = gclient_utils.IsUsingGit(self.root_dir(), entries.keys())
667 self._RunHooks(command, file_list, is_using_git)
668 624
669 if command == 'update': 625 if command == 'update':
670 # Notify the user if there is an orphaned entry in their working copy. 626 # Notify the user if there is an orphaned entry in their working copy.
671 # Only delete the directory if there are no changes in it, and 627 # Only delete the directory if there are no changes in it, and
672 # delete_unversioned_trees is set to true. 628 # delete_unversioned_trees is set to true.
673 prev_entries = self._ReadEntries() 629 entries = [i.name for i in self.tree(False)]
674 for entry in prev_entries: 630 for entry, prev_url in self._ReadEntries().iteritems():
675 # Fix path separator on Windows. 631 # Fix path separator on Windows.
676 entry_fixed = entry.replace('/', os.path.sep) 632 entry_fixed = entry.replace('/', os.path.sep)
677 e_dir = os.path.join(self.root_dir(), entry_fixed) 633 e_dir = os.path.join(self.root_dir(), entry_fixed)
678 # Use entry and not entry_fixed there. 634 # Use entry and not entry_fixed there.
679 if entry not in entries and os.path.exists(e_dir): 635 if entry not in entries and os.path.exists(e_dir):
680 modified_files = False 636 file_list = []
681 if isinstance(prev_entries, list): 637 scm = gclient_scm.CreateSCM(prev_url, self.root_dir(), entry_fixed)
682 # old .gclient_entries format was list, now dict 638 scm.status(self._options, [], file_list)
683 modified_files = gclient_scm.scm.SVN.CaptureStatus(e_dir) 639 modified_files = file_list != []
684 else:
685 file_list = []
686 scm = gclient_scm.CreateSCM(prev_entries[entry], self.root_dir(),
687 entry_fixed)
688 scm.status(self._options, [], file_list)
689 modified_files = file_list != []
690 if not self._options.delete_unversioned_trees or modified_files: 640 if not self._options.delete_unversioned_trees or modified_files:
691 # There are modified files in this entry. Keep warning until 641 # There are modified files in this entry. Keep warning until
692 # removed. 642 # removed.
693 print(('\nWARNING: \'%s\' is no longer part of this client. ' 643 print(('\nWARNING: \'%s\' is no longer part of this client. '
694 'It is recommended that you manually remove it.\n') % 644 'It is recommended that you manually remove it.\n') %
695 entry_fixed) 645 entry_fixed)
696 else: 646 else:
697 # Delete the entry 647 # Delete the entry
698 print('\n________ deleting \'%s\' in \'%s\'' % ( 648 print('\n________ deleting \'%s\' in \'%s\'' % (
699 entry_fixed, self.root_dir())) 649 entry_fixed, self.root_dir()))
700 gclient_utils.RemoveDirectory(e_dir) 650 gclient_utils.RemoveDirectory(e_dir)
701 # record the current list of entries for next time 651 # record the current list of entries for next time
702 self._SaveEntries(entries) 652 self._SaveEntries()
703 return 0 653 return 0
704 654
705 def PrintRevInfo(self): 655 def PrintRevInfo(self):
706 """Output revision info mapping for the client and its dependencies. 656 """Output revision info mapping for the client and its dependencies.
707 657
708 This allows the capture of an overall "revision" for the source tree that 658 This allows the capture of an overall "revision" for the source tree that
709 can be used to reproduce the same tree in the future. It is only useful for 659 can be used to reproduce the same tree in the future. It is only useful for
710 "unpinned dependencies", i.e. DEPS/deps references without a svn revision 660 "unpinned dependencies", i.e. DEPS/deps references without a svn revision
711 number or a git hash. A git branch name isn't "pinned" since the actual 661 number or a git hash. A git branch name isn't "pinned" since the actual
712 commit can change. 662 commit can change.
713 663
714 The --snapshot option allows creating a .gclient file to reproduce the tree. 664 The --snapshot option allows creating a .gclient file to reproduce the tree.
715 """ 665 """
716 if not self.dependencies: 666 if not self.dependencies:
717 raise gclient_utils.Error('No solution specified') 667 raise gclient_utils.Error('No solution specified')
668 # Load all the settings.
669 self.RunCommandRecursively(self._options, {}, None, [], None)
718 670
719 # Inner helper to generate base url and rev tuple
720 def GetURLAndRev(name, original_url): 671 def GetURLAndRev(name, original_url):
672 """Returns the revision-qualified SCM url."""
721 url, _ = gclient_utils.SplitUrlRevision(original_url) 673 url, _ = gclient_utils.SplitUrlRevision(original_url)
722 scm = gclient_scm.CreateSCM(original_url, self.root_dir(), name) 674 scm = gclient_scm.CreateSCM(original_url, self.root_dir(), name)
723 return (url, scm.revinfo(self._options, [], None)) 675 if not os.path.isdir(scm.checkout_path):
676 return None
677 return '%s@%s' % (url, scm.revinfo(self._options, [], None))
724 678
725 # text of the snapshot gclient file
726 new_gclient = ""
727 # Dictionary of { path : SCM url } to ensure no duplicate solutions
728 solution_names = {}
729 entries = {}
730 # Run on the base solutions first.
731 for solution in self.dependencies:
732 # Dictionary of { path : SCM url } to describe the gclient checkout
733 name = solution.name
734 if name in solution_names:
735 raise gclient_utils.Error("solution %s specified more than once" % name)
736 (url, rev) = GetURLAndRev(name, solution.url)
737 entries[name] = "%s@%s" % (url, rev)
738 solution_names[name] = "%s@%s" % (url, rev)
739
740 # Process the dependencies next (sort alphanumerically to ensure that
741 # containing directories get populated first and for readability)
742 deps = self._ParseAllDeps(entries)
743 deps_to_process = deps.keys()
744 deps_to_process.sort()
745
746 # First pass for direct dependencies.
747 for d in deps_to_process:
748 if type(deps[d]) == str:
749 (url, rev) = GetURLAndRev(d, deps[d])
750 entries[d] = "%s@%s" % (url, rev)
751
752 # Second pass for inherited deps (via the From keyword)
753 for d in deps_to_process:
754 if isinstance(deps[d], self.FromImpl):
755 deps_parent_url = entries[deps[d].module_name]
756 if deps_parent_url.find("@") < 0:
757 raise gclient_utils.Error("From %s missing revisioned url" %
758 deps[d].module_name)
759 sub_deps_base_url = deps[deps[d].module_name]
760 sub_deps = Dependency(self, deps[d].module_name, sub_deps_base_url
761 ).ParseDepsFile(False)
762 url = deps[d].GetUrl(d, sub_deps_base_url, self.root_dir(), sub_deps)
763 (url, rev) = GetURLAndRev(d, url)
764 entries[d] = "%s@%s" % (url, rev)
765
766 # Build the snapshot configuration string
767 if self._options.snapshot: 679 if self._options.snapshot:
768 url = entries.pop(name) 680 new_gclient = ''
769 custom_deps = ''.join([' \"%s\": \"%s\",\n' % (x, entries[x]) 681 # First level at .gclient
770 for x in sorted(entries.keys())]) 682 for d in self.dependencies:
771 683 entries = {}
772 new_gclient += self.DEFAULT_SNAPSHOT_SOLUTION_TEXT % { 684 def GrabDeps(sol):
773 'solution_name': name, 685 """Recursively grab dependencies."""
774 'solution_url': url, 686 for i in sol.dependencies:
775 'safesync_url' : '', 687 entries[i.name] = GetURLAndRev(i.name, i.parsed_url)
776 'solution_deps': custom_deps, 688 GrabDeps(i)
777 } 689 GrabDeps(d)
690 custom_deps = []
691 for k in sorted(entries.keys()):
692 if entries[k]:
693 # Quotes aren't escaped...
694 custom_deps.append(' \"%s\": \'%s\',\n' % (k, entries[k]))
695 else:
696 custom_deps.append(' \"%s\": None,\n' % k)
697 new_gclient += self.DEFAULT_SNAPSHOT_SOLUTION_TEXT % {
698 'solution_name': d.name,
699 'solution_url': d.url,
700 'safesync_url' : d.safesync_url or '',
701 'solution_deps': ''.join(custom_deps),
702 }
703 # Print the snapshot configuration file
704 print(self.DEFAULT_SNAPSHOT_FILE_TEXT % {'solution_list': new_gclient})
778 else: 705 else:
779 print(';\n'.join(['%s: %s' % (x, entries[x]) 706 entries = sorted(self.tree(False), key=lambda i: i.name)
780 for x in sorted(entries.keys())])) 707 for entry in entries:
781 708 revision = GetURLAndRev(entry.name, entry.parsed_url)
782 # Print the snapshot configuration file 709 line = '%s: %s' % (entry.name, revision)
783 if self._options.snapshot: 710 if not entry is entries[-1]:
784 config = self.DEFAULT_SNAPSHOT_FILE_TEXT % {'solution_list': new_gclient} 711 line += ';'
785 snapclient = GClient(self.root_dir(), self._options) 712 print line
786 snapclient.SetConfig(config)
787 print(snapclient.config_content)
788 713
789 def ParseDepsFile(self, direct_reference): 714 def ParseDepsFile(self, direct_reference):
790 """No DEPS to parse for a .gclient file.""" 715 """No DEPS to parse for a .gclient file."""
791 self.direct_reference = direct_reference 716 self.direct_reference = direct_reference
792 self.deps_parsed = True 717 self.deps_parsed = True
793 718
794 def root_dir(self): 719 def root_dir(self):
795 """Root directory of gclient checkout.""" 720 """Root directory of gclient checkout."""
796 return self._root_dir 721 return self._root_dir
797 722
(...skipping 300 matching lines...) Expand 10 before | Expand all | Expand 10 after
1098 'unpinned dependencies', i.e. DEPS/deps references without a svn revision 1023 'unpinned dependencies', i.e. DEPS/deps references without a svn revision
1099 number or a git hash. A git branch name isn't 'pinned' since the actual 1024 number or a git hash. A git branch name isn't 'pinned' since the actual
1100 commit can change. 1025 commit can change.
1101 """ 1026 """
1102 parser.add_option('--deps', dest='deps_os', metavar='OS_LIST', 1027 parser.add_option('--deps', dest='deps_os', metavar='OS_LIST',
1103 help='override deps for the specified (comma-separated) ' 1028 help='override deps for the specified (comma-separated) '
1104 'platform(s); \'all\' will process all deps_os ' 1029 'platform(s); \'all\' will process all deps_os '
1105 'references') 1030 'references')
1106 parser.add_option('-s', '--snapshot', action='store_true', 1031 parser.add_option('-s', '--snapshot', action='store_true',
1107 help='creates a snapshot .gclient file of the current ' 1032 help='creates a snapshot .gclient file of the current '
1108 'version of all repositories to reproduce the tree, ' 1033 'version of all repositories to reproduce the tree')
1109 'implies -a')
1110 (options, args) = parser.parse_args(args) 1034 (options, args) = parser.parse_args(args)
1111 client = GClient.LoadCurrentConfig(options) 1035 client = GClient.LoadCurrentConfig(options)
1112 if not client: 1036 if not client:
1113 raise gclient_utils.Error('client not configured; see \'gclient config\'') 1037 raise gclient_utils.Error('client not configured; see \'gclient config\'')
1114 client.PrintRevInfo() 1038 client.PrintRevInfo()
1115 return 0 1039 return 0
1116 1040
1117 1041
1118 def Command(name): 1042 def Command(name):
1119 return getattr(sys.modules[__name__], 'CMD' + name, None) 1043 return getattr(sys.modules[__name__], 'CMD' + name, None)
(...skipping 71 matching lines...) Expand 10 before | Expand all | Expand 10 after
1191 return CMDhelp(parser, argv) 1115 return CMDhelp(parser, argv)
1192 except gclient_utils.Error, e: 1116 except gclient_utils.Error, e:
1193 print >> sys.stderr, 'Error: %s' % str(e) 1117 print >> sys.stderr, 'Error: %s' % str(e)
1194 return 1 1118 return 1
1195 1119
1196 1120
1197 if '__main__' == __name__: 1121 if '__main__' == __name__:
1198 sys.exit(Main(sys.argv[1:])) 1122 sys.exit(Main(sys.argv[1:]))
1199 1123
1200 # vim: ts=2:sw=2:tw=80:et: 1124 # vim: ts=2:sw=2:tw=80:et:
OLDNEW
« no previous file with comments | « no previous file | gclient_utils.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698