GYP: Make GYP build deterministic

pylib/gyp/generator/ninja.py

 # Copyright (c) 2013 Google Inc. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import collections
 import copy
 import hashlib
 import json
 import multiprocessing
 import os.path
@@ skipping 51 matching lines @@
 generator_extra_sources_for_rules = []
 generator_filelist_paths = None
 
 generator_supports_multiple_toolsets = gyp.common.CrossCompileRequested()
 
 def StripPrefix(arg, prefix):
   if arg.startswith(prefix):
     return arg[len(prefix):]
   return arg
 
+def OrderDeterministically(l, **kwargs):
+  """ Sorts l so that it is ordered deterministically. """
+  return sorted(l, **kwargs)

[mithro-old, 2015/12/10 03:15:27]

I think this should be in gyp.common with OrderedSet?

[Zachary Forman, 2015/12/30 02:18:43]

Yeah, you caught me mid refactor. Good spot for it.

 
 def QuoteShellArgument(arg, flavor):
   """Quote a string such that it will be interpreted as a single argument
   by the shell."""
   # Rather than attempting to enumerate the bad shell characters, just
   # whitelist common OK ones and quote anything else.
   if re.match(r'^[a-zA-Z0-9_=.\\/-]+$', arg):
     return arg  # No quoting necessary.
   if flavor == 'win':
     return gyp.msvs_emulation.QuoteForRspFile(arg)
@@ skipping 332 matching lines @@
     # TODO(evan): it is rather confusing which things are lists and which
     # are strings.  Fix these.
     if 'dependencies' in spec:
       for dep in spec['dependencies']:
         if dep in self.target_outputs:
           target = self.target_outputs[dep]
           actions_depends.append(target.PreActionInput(self.flavor))
           compile_depends.append(target.PreCompileInput())
       actions_depends = filter(None, actions_depends)
       compile_depends = filter(None, compile_depends)
-      actions_depends = self.WriteCollapsedDependencies('actions_depends',
-                                                        actions_depends)
-      compile_depends = self.WriteCollapsedDependencies('compile_depends',
-                                                        compile_depends)
+      actions_depends = self.WriteCollapsedDependencies(
+          'actions_depends', OrderDeterministically(actions_depends))
+      compile_depends = self.WriteCollapsedDependencies(
+          'compile_depends', OrderDeterministically(compile_depends))
       self.target.preaction_stamp = actions_depends
       self.target.precompile_stamp = compile_depends
 
     # Write out actions, rules, and copies.  These must happen before we
     # compile any sources, so compute a list of predependencies for sources
     # while we do it.
     extra_sources = []
     mac_bundle_depends = []
     self.target.actions_stamp = self.WriteActionsRulesCopies(
         spec, extra_sources, actions_depends, mac_bundle_depends)
@@ skipping 114 matching lines @@
     if 'rules' in spec:
       outputs += self.WriteRules(spec['rules'], extra_sources, prebuild,
                                  mac_bundle_resources,
                                  extra_mac_bundle_resources)
     if 'copies' in spec:
       outputs += self.WriteCopies(spec['copies'], prebuild, mac_bundle_depends)
 
     if 'sources' in spec and self.flavor == 'win':
       outputs += self.WriteWinIdlFiles(spec, prebuild)
 
-    stamp = self.WriteCollapsedDependencies('actions_rules_copies', outputs)
+    stamp = self.WriteCollapsedDependencies('actions_rules_copies',
+                                            OrderDeterministically(outputs))
 
     if self.is_mac_bundle:
       xcassets = self.WriteMacBundleResources(
           extra_mac_bundle_resources + mac_bundle_resources, mac_bundle_depends)
       partial_info_plist = self.WriteMacXCassets(xcassets, mac_bundle_depends)
       self.WriteMacInfoPlist(partial_info_plist, mac_bundle_depends)
 
     return stamp
 
   def GenerateDescription(self, verb, message, fallback):
@@ skipping 77 matching lines @@
       # Rules can potentially make use of some special variables which
       # must vary per source file.
       # Compute the list of variables we'll need to provide.
       special_locals = ('source', 'root', 'dirname', 'ext', 'name')
       needed_variables = set(['source'])
       for argument in args:
         for var in special_locals:
           if '${%s}' % var in argument:
             needed_variables.add(var)
 
+      needed_variables = OrderDeterministically(needed_variables)
+
       def cygwin_munge(path):
         # pylint: disable=cell-var-from-loop
         if is_cygwin:
           return path.replace('\\', '/')
         return path
 
       inputs = [self.GypPathToNinja(i, env) for i in rule.get('inputs', [])]
 
       # If there are n source files matching the rule, and m additional rule
       # inputs, then adding 'inputs' to each build edge written below will
       # write m * n inputs. Collapsing reduces this to m + n.
       sources = rule.get('rule_sources', [])
       num_inputs = len(inputs)
       if prebuild:
         num_inputs += 1
       if num_inputs > 2 and len(sources) > 2:
         inputs = [self.WriteCollapsedDependencies(
-          rule['rule_name'], inputs, order_only=prebuild)]
+          rule['rule_name'], OrderDeterministically(inputs), order_only=prebuild)]
         prebuild = []
 
       # For each source file, write an edge that generates all the outputs.
       for source in sources:
         source = os.path.normpath(source)
         dirname, basename = os.path.split(source)
         root, ext = os.path.splitext(basename)
 
         # Gather the list of inputs and outputs, expanding $vars if possible.
         outputs = [self.ExpandRuleVariables(o, root, dirname,
@@ skipping 34 matching lines @@
           elif var == 'name':
             extra_bindings.append(('name', cygwin_munge(basename)))
           else:
             assert var == None, repr(var)
 
         outputs = [self.GypPathToNinja(o, env) for o in outputs]
         if self.flavor == 'win':
           # WriteNewNinjaRule uses unique_name for creating an rsp file on win.
           extra_bindings.append(('unique_name',
               hashlib.md5(outputs[0]).hexdigest()))
+
+        # Make sure we sort extra_bindings so that output is deterministic.
         self.ninja.build(outputs, rule_name, self.GypPathToNinja(source),
                          implicit=inputs,
                          order_only=prebuild,
                          variables=extra_bindings)
 
         all_outputs.extend(outputs)
 
     return all_outputs
 
   def WriteCopies(self, copies, prebuild, mac_bundle_depends):
@@ skipping 385 matching lines @@
             new_deps = [target.binary]
           for new_dep in new_deps:
             if new_dep not in extra_link_deps:
               extra_link_deps.add(new_dep)
               link_deps.append(new_dep)
 
         final_output = target.FinalOutput()
         if not linkable or final_output != target.binary:
           implicit_deps.add(final_output)
 
+    link_deps = OrderDeterministically(link_deps)
+
     extra_bindings = []
     if self.uses_cpp and self.flavor != 'win':
       extra_bindings.append(('ld', '$ldxx'))
 
     output = self.ComputeOutput(spec, arch)
     if arch is None and not self.is_mac_bundle:
       self.AppendPostbuildVariable(extra_bindings, spec, output, output)
 
     is_executable = spec['type'] == 'executable'
     # The ldflags config key is not used on mac or win. On those platforms
@@ skipping 96 matching lines @@
         output = [output, output + '.TOC']
       else:
         command = command + '_notoc'
     elif self.flavor == 'win':
       extra_bindings.append(('binary', output))
       pdbname = self.msvs_settings.GetPDBName(
           config_name, self.ExpandSpecial, output + '.pdb')
       if pdbname:
         output = [output, pdbname]
 
-
     if len(solibs):
-      extra_bindings.append(('solibs', gyp.common.EncodePOSIXShellList(solibs)))
+      extra_bindings.append(('solibs',
+          gyp.common.EncodePOSIXShellList(OrderDeterministically(solibs))))
 
     ninja_file.build(output, command + command_suffix, link_deps,
-                     implicit=list(implicit_deps),
-                     order_only=list(order_deps),
+                     implicit=OrderDeterministically(implicit_deps),
+                     order_only=order_deps,
                      variables=extra_bindings)
     return linked_binary
 
   def WriteTarget(self, spec, config_name, config, link_deps, compile_deps):
     extra_link_deps = any(self.target_outputs.get(dep).Linkable()
                           for dep in spec.get('dependencies', [])
                           if dep in self.target_outputs)
     if spec['type'] == 'none' or (not link_deps and not extra_link_deps):
       # TODO(evan): don't call this function for 'none' target types, as
       # it doesn't do anything, and we fake out a 'binary' with a stamp file.
@@ skipping 1060 matching lines @@
       non_empty_target_names.add(name)
     else:
       empty_target_names.add(name)
 
   if target_short_names:
     # Write a short name to build this target.  This benefits both the
     # "build chrome" case as well as the gyp tests, which expect to be
     # able to run actions and build libraries by their short name.
     master_ninja.newline()
     master_ninja.comment('Short names for targets.')
-    for short_name in target_short_names:
-      master_ninja.build(short_name, 'phony', [x.FinalOutput() for x in
-                                               target_short_names[short_name]])
+
+    for short_name in OrderDeterministically(target_short_names):
+      master_ninja.build(short_name, 'phony', ([x.FinalOutput() for x in
+                                               target_short_names[short_name]]))
 
   # Write phony targets for any empty targets that weren't written yet. As
   # short names are  not necessarily unique only do this for short names that
   # haven't already been output for another target.
   empty_target_names = empty_target_names - non_empty_target_names
   if empty_target_names:
     master_ninja.newline()
     master_ninja.comment('Empty targets (output for completeness).')
-    for name in sorted(empty_target_names):
+
+    # Iterate over empty target names in a deterministic order by sorting.
+    for name in OrderDeterministically(empty_target_names):
       master_ninja.build(name, 'phony')
 
   if all_outputs:
     master_ninja.newline()
-    master_ninja.build('all', 'phony', list(all_outputs))
+    master_ninja.build('all', 'phony', OrderDeterministically(all_outputs))
     master_ninja.default(generator_flags.get('default_target', 'all'))
 
   master_ninja_file.close()
 
 
 def PerformBuild(data, configurations, params):
   options = params['options']
   for config in configurations:
     builddir = os.path.join(options.toplevel_dir, 'out', config)
     arguments = ['ninja', '-C', builddir]
@@ skipping 34 matching lines @@
           arglists.append(
               (target_list, target_dicts, data, params, config_name))
         pool.map(CallGenerateOutputForConfig, arglists)
       except KeyboardInterrupt, e:
         pool.terminate()
         raise e
     else:
       for config_name in config_names:
         GenerateOutputForConfig(target_list, target_dicts, data, params,
                                 config_name)