Stop modifiying requirements out of thread and generate it instead.

gclient.py

 #!/usr/bin/env python
 # Copyright (c) 2011 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Meta checkout manager supporting both Subversion and GIT.
 
 Files
   .gclient      : Current client configuration, written by 'config' command.
                   Format is a Python script defining 'solutions', a list whose
@@ skipping 243 matching lines @@
     # dependency, i.e. its DEPS wasn't read.
     self._deps_parsed = False
     # This dependency has been processed, i.e. checked out
     self._processed = False
     # This dependency had its hook run
     self._hooks_ran = False
 
     if not self.name and self.parent:
       raise gclient_utils.Error('Dependency without name')
 
-  def setup_requirements(self):
-    """Setup self.requirements and find any other dependency who would have self
-    as a requirement.
-
-    Returns True if this entry should be added, False if it is a duplicate of
-    another entry.
-    """
-    if self.name in [s.name for s in self.parent.dependencies]:
-      raise gclient_utils.Error(
-          'The same name "%s" appears multiple times in the deps section' %
-              self.name)
-    if self.should_process:
-      siblings = [d for d in self.root.subtree(False) if d.name == self.name]
-      for sibling in siblings:
-        if self.url != sibling.url:
-          raise gclient_utils.Error(
-              'Dependency %s specified more than once:\n  %s\nvs\n  %s' %
-              (self.name, sibling.hierarchy(), self.hierarchy()))
-        # In theory we could keep it as a shadow of the other one. In
-        # practice, simply ignore it.
-        logging.warn('Won\'t process duplicate dependency %s' % sibling)
-        return False
-
+  @property
+  def requirements(self):
+    """Calculate the list of requirements."""
+    requirements = set()
     # self.parent is implicitly a requirement. This will be recursive by
     # definition.
     if self.parent and self.parent.name:
-      self.add_requirement(self.parent.name)
+      requirements.add(self.parent.name)
 
     # For a tree with at least 2 levels*, the leaf node needs to depend
     # on the level higher up in an orderly way.
     # This becomes messy for >2 depth as the DEPS file format is a dictionary,
     # thus unsorted, while the .gclient format is a list thus sorted.
     #
     # * _recursion_limit is hard coded 2 and there is no hope to change this
     # value.
     #
     # Interestingly enough, the following condition only works in the case we
     # want: self is a 2nd level node. 3nd level node wouldn't need this since
     # they already have their parent as a requirement.
-    root_deps = self.root.dependencies
-    if self.parent in root_deps:
-      for i in root_deps:
-        if i is self.parent:
-          break
-        if i.name:
-          self.add_requirement(i.name)
+    if self.parent and self.parent.parent and not self.parent.parent.parent:
+      requirements |= set(i.name for i in self.root.dependencies if i.name)
 
     if isinstance(self.url, self.FromImpl):
-      self.add_requirement(self.url.module_name)
+      requirements.add(self.url.module_name)
 
-    if self.name and self.should_process:
-      for obj in self.root.depth_first_tree():
-        if obj is self or not obj.name:
-          continue
-        # Step 1: Find any requirements self may need.
-        if self.name.startswith(posixpath.join(obj.name, '')):
-          self.add_requirement(obj.name)
-        # Step 2: Find any requirements self may impose.
-        if obj.name.startswith(posixpath.join(self.name, '')):
-          obj.add_requirement(self.name)
+    if self.name:
+      requirements |= set(
+          obj.name for obj in self.root.subtree(False)
+          if (obj is not self
+              and obj.name and
+              self.name.startswith(posixpath.join(obj.name, ''))))
+    requirements = tuple(sorted(requirements))
+    logging.info('Dependency(%s).requirements = %s' % (self.name, requirements))
+    return requirements
+
+  def verify_validity(self):
+    """Verifies that this Dependency is fine to add as a child of another one.
+
+    Returns True if this entry should be added, False if it is a duplicate of
+    another entry.
+    """
+    logging.info('Dependency(%s).verify_validity()' % self.name)
+    if self.name in [s.name for s in self.parent.dependencies]:
+      raise gclient_utils.Error(
+          'The same name "%s" appears multiple times in the deps section' %
+              self.name)
+    if not self.should_process:
+      # Return early, no need to set requirements.
+      return True
+
+    # This require a full tree traversal with locks.
+    siblings = [d for d in self.root.subtree(False) if d.name == self.name]
+    for sibling in siblings:
+      if self.url != sibling.url:
+        raise gclient_utils.Error(
+            'Dependency %s specified more than once:\n  %s\nvs\n  %s' %
+            (self.name, sibling.hierarchy(), self.hierarchy()))
+      # In theory we could keep it as a shadow of the other one. In
+      # practice, simply ignore it.
+      logging.warn('Won\'t process duplicate dependency %s' % sibling)
+      return False
     return True
 
   def LateOverride(self, url):
     """Resolves the parsed url from url.
 
     Manages From() keyword accordingly. Do not touch self.parsed_url nor
     self.url because it may called with other urls due to From()."""
     assert self.parsed_url == None or not self.should_process, self.parsed_url
     parsed_url = self.get_custom_deps(self.name, url)
     if parsed_url != url:
-      logging.info('LateOverride(%s, %s) -> %s' % (self.name, url, parsed_url))
+      logging.info(
+          'Dependency(%s).LateOverride(%s) -> %s' %
+          (self.name, url, parsed_url))
       return parsed_url
 
     if isinstance(url, self.FromImpl):
+      # Requires tree traversal.
       ref = [
           dep for dep in self.root.subtree(True) if url.module_name == dep.name
       ]
       if not ref:
         raise gclient_utils.Error('Failed to find one reference to %s. %s' % (
             url.module_name, ref))
       # It may happen that len(ref) > 1 but it's no big deal.
       ref = ref[0]
       sub_target = url.sub_target_name or self.name
       found_deps = [d for d in ref.dependencies if d.name == sub_target]
       if len(found_deps) != 1:
         raise gclient_utils.Error(
             'Couldn\'t find %s in %s, referenced by %s (parent: %s)\n%s' % (
                 sub_target, ref.name, self.name, self.parent.name,
                 str(self.root)))
 
       # Call LateOverride() again.
       found_dep = found_deps[0]
       parsed_url = found_dep.LateOverride(found_dep.url)
       logging.info(
-          'LateOverride(%s, %s) -> %s (From)' % (self.name, url, parsed_url))
+          'Dependency(%s).LateOverride(%s) -> %s (From)' %
+          (self.name, url, parsed_url))
       return parsed_url
 
     if isinstance(url, basestring):
       parsed_url = urlparse.urlparse(url)
       if not parsed_url[0]:
         # A relative url. Fetch the real base.
         path = parsed_url[2]
         if not path.startswith('/'):
           raise gclient_utils.Error(
               'relative DEPS entry \'%s\' must begin with a slash' % url)
         # Create a scm just to query the full url.
         parent_url = self.parent.parsed_url
         if isinstance(parent_url, self.FileImpl):
           parent_url = parent_url.file_location
         scm = gclient_scm.CreateSCM(parent_url, self.root.root_dir, None)
         parsed_url = scm.FullUrlForRelativeUrl(url)
       else:
         parsed_url = url
-      logging.info('LateOverride(%s, %s) -> %s' % (self.name, url, parsed_url))
+      logging.info(
+          'Dependency(%s).LateOverride(%s) -> %s' %
+          (self.name, url, parsed_url))
       return parsed_url
 
     if isinstance(url, self.FileImpl):
-      logging.info('LateOverride(%s, %s) -> %s (File)' % (self.name, url, url))
+      logging.info(
+          'Dependency(%s).LateOverride(%s) -> %s (File)' %
+          (self.name, url, url))
       return url
 
     if url is None:
-      logging.info('LateOverride(%s, %s) -> %s' % (self.name, url, url))
+      logging.info(
+          'Dependency(%s).LateOverride(%s) -> %s' % (self.name, url, url))
       return url
 
     raise gclient_utils.Error('Unknown url type')
 
   def ParseDepsFile(self):
     """Parses the DEPS file for this dependency."""
     assert not self.deps_parsed
     assert not self.dependencies
     # One thing is unintuitive, vars = {} must happen before Var() use.
     local_scope = {}
@@ skipping 56 matching lines @@
     for name, url in deps.iteritems():
       should_process = self.recursion_limit and self.should_process
       deps_to_add.append(Dependency(
           self, name, url, None, None, None, None,
           self.deps_file, should_process))
     self.add_dependencies_and_close(deps_to_add, local_scope.get('hooks', []))
     logging.info('ParseDepsFile(%s) done' % self.name)
 
   def add_dependencies_and_close(self, deps_to_add, hooks):
     """Adds the dependencies, hooks and mark the parsing as done."""
-    for dep in deps_to_add:
-      if dep.setup_requirements():
+    for dep in sorted(deps_to_add, key=lambda x: x.name):
+      if dep.verify_validity():
         self.add_dependency(dep)
     self._mark_as_parsed(hooks)
 
   @staticmethod
   def maybeGetParentRevision(
       command, options, parsed_url, parent_name, revision_overrides):
     """If we are performing an update and --transitive is set, set the
     revision to the parent's revision. If we have an explicit revision
     do nothing."""
     if command == 'update' and options.transitive and not options.revision:
@@ skipping 24 matching lines @@
       revision = gclient_utils.MakeDateRevision(revision_date)
       if options.verbose:
         print("Updating revision override from %s to %s." %
               (options.revision, revision))
       revision_overrides[name] = revision
 
   # Arguments number differs from overridden method
   # pylint: disable=W0221
   def run(self, revision_overrides, command, args, work_queue, options):
     """Runs |command| then parse the DEPS file."""
+    logging.info('Dependency(%s).run()' % self.name)
     assert self._file_list == []
     if not self.should_process:
       return
     # When running runhooks, there's no need to consult the SCM.
     # All known hooks are expected to run unconditionally regardless of working
     # copy state, so skip the SCM status check.
     run_scm = command not in ('runhooks', None)
     parsed_url = self.LateOverride(self.url)
     file_list = []
     if run_scm and parsed_url:
@@ skipping 963 matching lines @@
   except (gclient_utils.Error, subprocess2.CalledProcessError), e:
     print >> sys.stderr, 'Error: %s' % str(e)
     return 1
 
 
 if '__main__' == __name__:
   fix_encoding.fix_encoding()
   sys.exit(Main(sys.argv[1:]))
 
 # vim: ts=2:sw=2:tw=80:et: