Reland: Add ability to override DEPS file in recursedeps

gclient.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Meta checkout manager supporting both Subversion and GIT."""
 # Files
 #   .gclient      : Current client configuration, written by 'config' command.
 #                   Format is a Python script defining 'solutions', a list whose
 #                   entries each are maps binding the strings "name" and "url"
@@ skipping 353 matching lines @@
     # unavailable
     self._got_revision = None
 
     # This is a mutable value that overrides the normal recursion limit for this
     # dependency.  It is read from the actual DEPS file so cannot be set on
     # class instantiation.
     self.recursion_override = None
     # recursedeps is a mutable value that selectively overrides the default
     # 'no recursion' setting on a dep-by-dep basis.  It will replace
     # recursion_override.
+    #
+    # It will be a dictionary of {deps_name: {"deps_file": depfile_name}} or
+    # None.
     self.recursedeps = None
 
     if not self.name and self.parent:
       raise gclient_utils.Error('Dependency without name')
 
   @property
   def requirements(self):
     """Calculate the list of requirements."""
     requirements = set()
     # self.parent is implicitly a requirement. This will be recursive by
@@ skipping 259 matching lines @@
           if not isinstance(val, (dict, list, tuple, str)):
             raise gclient_utils.Error(
               'ParseDepsFile(%s): Strict mode disallows %r -> %r' %
               (self.name, key, val))
 
     deps = local_scope.get('deps', {})
     if 'recursion' in local_scope:
       self.recursion_override = local_scope.get('recursion')
       logging.warning(
           'Setting %s recursion to %d.', self.name, self.recursion_limit)
-    self.recursedeps = local_scope.get('recursedeps', None)
+    self.recursedeps = None
     if 'recursedeps' in local_scope:
-      self.recursedeps = set(self.recursedeps)
+      self.recursedeps = {}
+      for ent in local_scope['recursedeps']:
+        if isinstance(ent, basestring):
+          self.recursedeps[ent] = {"deps_file": self.deps_file}
+        else:  # (depname, depsfilename)
+          self.recursedeps[ent[0]] = {"deps_file": ent[1]}

[Dirk Pranke, 2016/05/04 22:49:59]

the document above only says that dicts are supported. Are we supporting strings
for backwards compatibility? If so, we should either note that in the doc, or
file a TODO bug to make sure to update any callers, or do both.

       logging.warning('Found recursedeps %r.', repr(self.recursedeps))
     # If present, save 'target_os' in the local_target_os property.
     if 'target_os' in local_scope:
       self.local_target_os = local_scope['target_os']
     # load os specific dependencies if defined.  these dependencies may
     # override or extend the values defined by the 'deps' member.
     target_os_list = self.target_os
     if 'deps_os' in local_scope and target_os_list:
       deps = self.MergeWithOsDeps(deps, local_scope['deps_os'], target_os_list)
 
@@ skipping 14 matching lines @@
       for d, url in deps.items():
         # normpath is required to allow DEPS to use .. in their
         # dependency local path.
         rel_deps[os.path.normpath(os.path.join(self.name, d))] = url
       logging.warning('Updating deps by prepending %s.', self.name)
       deps = rel_deps
 
       # Update recursedeps if it's set.
       if self.recursedeps is not None:
         logging.warning('Updating recursedeps by prepending %s.', self.name)
-        rel_deps = set()
-        for d in self.recursedeps:
-          rel_deps.add(os.path.normpath(os.path.join(self.name, d)))
+        rel_deps = {}
+        for depname, options in self.recursedeps.iteritems():
+          rel_deps[os.path.normpath(os.path.join(self.name, depname))] = options
         self.recursedeps = rel_deps
 
     if 'allowed_hosts' in local_scope:
       try:
         self._allowed_hosts = frozenset(local_scope.get('allowed_hosts'))
       except TypeError:  # raised if non-iterable
         pass
       if not self._allowed_hosts:
         logging.warning("allowed_hosts is specified but empty %s",
                         self._allowed_hosts)
         raise gclient_utils.Error(
             'ParseDepsFile(%s): allowed_hosts must be absent '
             'or a non-empty iterable' % self.name)
 
     # Convert the deps into real Dependency.
     deps_to_add = []
     for name, url in deps.iteritems():
       should_process = self.recursion_limit and self.should_process
+      deps_file = self.deps_file
+      if self.recursedeps is not None:
+        ent = self.recursedeps.get(name)
+        if ent is not None:
+          deps_file = ent['deps_file']
       deps_to_add.append(Dependency(
           self, name, url, None, None, None, self.custom_vars, None,
-          self.deps_file, should_process))

[iannucci, 2016/05/03 22:39:29]

The existing behavior ALWAYS assumed self.deps_file. In patchset 2 we now use
that as the default value when recursedeps is specified. Patchset 1 worked for
non-infra repos because only the infra repos use recursedeps right now AND use
non-default DEPS files.

+          deps_file, should_process))
     deps_to_add.sort(key=lambda x: x.name)
 
     # override named sets of hooks by the custom hooks
     hooks_to_run = []
     hook_names_to_suppress = [c.get('name', '') for c in self.custom_hooks]
     for hook in local_scope.get('hooks', []):
       if hook.get('name', '') not in hook_names_to_suppress:
         hooks_to_run.append(hook)
 
     # add the replacements and any additions
@@ skipping 1605 matching lines @@
 
 
 if '__main__' == __name__:
   try:
     sys.exit(main(sys.argv[1:]))
   except KeyboardInterrupt:
     sys.stderr.write('interrupted\n')
     sys.exit(1)
 
 # vim: ts=2:sw=2:tw=80:et: