Fix parallelization of multiple solutions.

gclient.py

 #!/usr/bin/env python
 # Copyright (c) 2011 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Meta checkout manager supporting both Subversion and GIT.
 
 Files
   .gclient      : Current client configuration, written by 'config' command.
                   Format is a Python script defining 'solutions', a list whose
@@ skipping 123 matching lines @@
       elif var_name in self._local_scope.get("vars", {}):
         return self._local_scope["vars"][var_name]
       raise gclient_utils.Error("Var is not defined: %s" % var_name)
 
 
 class Dependency(GClientKeywords, gclient_utils.WorkItem):
   """Object that represents a dependency checkout."""
 
   def __init__(self, parent, name, url, safesync_url, custom_deps,
                custom_vars, deps_file, should_process):
+    # Warning: this function can be called from any thread. Both
+    # self.dependencies and self.requirements are read and modified from
+    # multiple threads at the same time. Sad.
     GClientKeywords.__init__(self)
     gclient_utils.WorkItem.__init__(self)
     self.parent = parent
     self.name = name
     self.url = url
     self.parsed_url = None
     # These 2 are only set in .gclient and not in DEPS files.
     self.safesync_url = safesync_url
     self.custom_vars = custom_vars or {}
     self.custom_deps = custom_deps or {}
     self.deps_hooks = []
     self.dependencies = []
     self.deps_file = deps_file
     # A cache of the files affected by the current operation, necessary for
     # hooks.
     self._file_list = []
     # If it is not set to True, the dependency wasn't processed for its child
     # dependency, i.e. its DEPS wasn't read.
     self.deps_parsed = False
     # This dependency should be processed, i.e. checked out
     self.should_process = should_process
     # This dependency has been processed, i.e. checked out
     self.processed = False
     # This dependency had its hook run
     self.hooks_ran = False
     # Required dependencies to run before running this one:
-    self.requirements = []
-    if self.parent and self.parent.name:
-      self.requirements.append(self.parent.name)
-    if isinstance(self.url, self.FromImpl):
-      self.requirements.append(self.url.module_name)
+    self.requirements = set()
+
+    self._FindDependencies()
 
     # Sanity checks
     if not self.name and self.parent:
       raise gclient_utils.Error('Dependency without name')
     if not isinstance(self.url,
         (basestring, self.FromImpl, self.FileImpl, None.__class__)):
       raise gclient_utils.Error('dependency url must be either a string, None, '
                                 'File() or From() instead of %s' %
                                 self.url.__class__.__name__)
     if '/' in self.deps_file or '\\' in self.deps_file:
       raise gclient_utils.Error('deps_file name must not be a path, just a '
                                 'filename. %s' % self.deps_file)
 
+  def _FindDependencies(self):
+    """Setup self.requirements and find any other dependency who would have self
+    as a requirement.
+    """
+    # self.parent is implicitly a requirement. This will be recursive by
+    # definition.
+    if self.parent and self.parent.name:
+      self.requirements.add(self.parent.name)
+
+    # For a solution with multiple items, dependencies from the non-primary
+    # solution will dependent on the prior solutions.
+    if self.parent and self.parent.parent:

[Dirk Pranke, 2011/08/30 21:09:36]

I'm a little lost here. How do you know that self.parent.parent refers to an
earlier solution?

[M-A Ruel, 2011/09/01 16:17:37]

Redid the doc since the code didn't have to be modified. I also fixed the code
to work with any _recursion_limit value.

+      for i in range(0, self.parent.parent.dependencies.index(self.parent)):
+        value = self.parent.parent.dependencies[i]
+        if value.name:
+          self.requirements.add(value.name)
+
+    if isinstance(self.url, self.FromImpl):
+      self.requirements.add(self.url.module_name)
+
+    if self.name:

[Dirk Pranke, 2011/08/30 21:09:36]

When is self.name null? Should that be legal?

[M-A Ruel, 2011/09/01 16:17:37]

The 'containing' root object has no name.

+      def all_objects(root):

[Dirk Pranke, 2011/08/30 21:09:36]

what is an object in this case? Is there a better name for this function?

[M-A Ruel, 2011/09/01 16:17:37]

Renamed to yield_full_tree().

+        """Depth-first recursion."""
+        yield root
+        for i in root.dependencies:
+          for j in all_objects(i):
+            yield j
+
+      for obj in all_objects(self.root_parent()):
+        if obj is self or not obj.name:
+          continue
+        # Step 1: Find any requirements self may need.
+        if self.name.startswith(posixpath.join(obj.name, '')):
+          self.requirements.add(obj.name)
+        # Step 2: Find any requirements self may impose.
+        if obj.name.startswith(posixpath.join(self.name, '')):
+          obj.requirements.add(self.name)
+
   def LateOverride(self, url):
     """Resolves the parsed url from url.
 
     Manages From() keyword accordingly. Do not touch self.parsed_url nor
     self.url because it may called with other urls due to From()."""
     assert self.parsed_url == None or not self.should_process, self.parsed_url
     overriden_url = self.get_custom_deps(self.name, url)
     if overriden_url != url:
       logging.info('%s, %s was overriden to %s' % (self.name, url,
           overriden_url))
@@ skipping 201 matching lines @@
         maybeGetParentRevision(options)
         scm = gclient_scm.CreateSCM(self.parsed_url, self.root_dir(), self.name)
         scm.RunCommand(command, options, args, self._file_list)
         maybeConvertToDateRevision(options)
         self._file_list = [os.path.join(self.name, f.strip())
                            for f in self._file_list]
     self.processed = True
     if self.recursion_limit() > 0:
       # Then we can parse the DEPS file.
       self.ParseDepsFile()
-      # Adjust the implicit dependency requirement; e.g. if a DEPS file contains
-      # both src/foo and src/foo/bar, src/foo/bar is implicitly dependent of
-      # src/foo. Yes, it's O(n^2)... It's important to do that before
-      # enqueueing them.
-      for s in self.dependencies:
-        for s2 in self.dependencies:
-          if s is s2:
-            continue
-          if s.name.startswith(posixpath.join(s2.name, '')):
-            s.requirements.append(s2.name)
 
       # Parse the dependencies of this dependency.
       for s in self.dependencies:
         work_queue.enqueue(s)
 
   def RunHooksRecursively(self, options):
     """Evaluates all hooks, running actions as needed. run()
     must have been called before to load the DEPS."""
     assert self.hooks_ran == False
     if not self.should_process or self.recursion_limit() <= 0:
@@ skipping 879 matching lines @@
   except (gclient_utils.Error, subprocess2.CalledProcessError), e:
     print >> sys.stderr, 'Error: %s' % str(e)
     return 1
 
 
 if '__main__' == __name__:
   fix_encoding.fix_encoding()
   sys.exit(Main(sys.argv[1:]))
 
 # vim: ts=2:sw=2:tw=80:et: