GYP: Make GYP build deterministic

pylib/gyp/generator/ninja.py

 # Copyright (c) 2013 Google Inc. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import collections
 import copy
 import hashlib
 import json
 import multiprocessing
 import os.path
 import re
 import signal
 import subprocess
 import sys
 import gyp
 import gyp.common
 from gyp.common import OrderedSet
+

[Nico, 2016/01/13 20:16:54]

why this newline?

[Zachary Forman, 2016/01/18 20:44:36]

Fixed

 import gyp.msvs_emulation
 import gyp.MSVSUtil as MSVSUtil
 import gyp.xcode_emulation
 from cStringIO import StringIO
 
 from gyp.common import GetEnvironFallback
 import gyp.ninja_syntax as ninja_syntax
 
 generator_default_variables = {
   'EXECUTABLE_PREFIX': '',
@@ skipping 621 matching lines @@
 
       # Rules can potentially make use of some special variables which
       # must vary per source file.
       # Compute the list of variables we'll need to provide.
       special_locals = ('source', 'root', 'dirname', 'ext', 'name')
       needed_variables = set(['source'])
       for argument in args:
         for var in special_locals:
           if '${%s}' % var in argument:
             needed_variables.add(var)
+      needed_variables = sorted(needed_variables)
 
       def cygwin_munge(path):
         # pylint: disable=cell-var-from-loop
         if is_cygwin:
           return path.replace('\\', '/')
         return path
 
       inputs = [self.GypPathToNinja(i, env) for i in rule.get('inputs', [])]
 
       # If there are n source files matching the rule, and m additional rule
@@ skipping 11 matching lines @@
       # For each source file, write an edge that generates all the outputs.
       for source in sources:
         source = os.path.normpath(source)
         dirname, basename = os.path.split(source)
         root, ext = os.path.splitext(basename)
 
         # Gather the list of inputs and outputs, expanding $vars if possible.
         outputs = [self.ExpandRuleVariables(o, root, dirname,
                                             source, ext, basename)
                    for o in rule['outputs']]
-

[Nico, 2016/01/13 20:16:54]

what was wrong with the 3 newlines here?

[Zachary Forman, 2016/01/18 20:44:36]

Reverted

         if int(rule.get('process_outputs_as_sources', False)):
           extra_sources += outputs
-
         was_mac_bundle_resource = source in mac_bundle_resources
         if was_mac_bundle_resource or \
             int(rule.get('process_outputs_as_mac_bundle_resources', False)):
           extra_mac_bundle_resources += outputs
           # Note: This is n_resources * n_outputs_in_rule.  Put to-be-removed
           # items in a set and remove them all in a single pass if this becomes
           # a performance issue.
           if was_mac_bundle_resource:
             mac_bundle_resources.remove(source)
-
         extra_bindings = []
         for var in needed_variables:
           if var == 'root':
             extra_bindings.append(('root', cygwin_munge(root)))
           elif var == 'dirname':
             # '$dirname' is a parameter to the rule action, which means
             # it shouldn't be converted to a Ninja path.  But we don't
             # want $!PRODUCT_DIR in there either.
             dirname_expanded = self.ExpandSpecial(dirname, self.base_to_build)
             extra_bindings.append(('dirname', cygwin_munge(dirname_expanded)))
           elif var == 'source':
             # '$source' is a parameter to the rule action, which means
             # it shouldn't be converted to a Ninja path.  But we don't
             # want $!PRODUCT_DIR in there either.
             source_expanded = self.ExpandSpecial(source, self.base_to_build)
             extra_bindings.append(('source', cygwin_munge(source_expanded)))
           elif var == 'ext':
             extra_bindings.append(('ext', ext))
           elif var == 'name':
             extra_bindings.append(('name', cygwin_munge(basename)))
           else:
             assert var == None, repr(var)
 
         outputs = [self.GypPathToNinja(o, env) for o in outputs]
         if self.flavor == 'win':
           # WriteNewNinjaRule uses unique_name for creating an rsp file on win.
           extra_bindings.append(('unique_name',
               hashlib.md5(outputs[0]).hexdigest()))
+
         self.ninja.build(outputs, rule_name, self.GypPathToNinja(source),
                          implicit=inputs,
                          order_only=prebuild,
                          variables=extra_bindings)
 
         all_outputs.extend(outputs)
 
     return all_outputs
 
   def WriteCopies(self, copies, prebuild, mac_bundle_depends):
@@ skipping 503 matching lines @@
         command = command + '_notoc'
     elif self.flavor == 'win':
       extra_bindings.append(('binary', output))
       pdbname = self.msvs_settings.GetPDBName(
           config_name, self.ExpandSpecial, output + '.pdb')
       if pdbname:
         output = [output, pdbname]
 
 
     if len(solibs):
-      extra_bindings.append(('solibs', gyp.common.EncodePOSIXShellList(solibs)))
+      extra_bindings.append(('solibs',
+          gyp.common.EncodePOSIXShellList(sorted(solibs))))
 
     ninja_file.build(output, command + command_suffix, link_deps,
-                     implicit=list(implicit_deps),
+                     implicit=sorted(implicit_deps),
                      order_only=list(order_deps),
                      variables=extra_bindings)
     return linked_binary
 
   def WriteTarget(self, spec, config_name, config, link_deps, compile_deps):
     extra_link_deps = any(self.target_outputs.get(dep).Linkable()
                           for dep in spec.get('dependencies', [])
                           if dep in self.target_outputs)
     if spec['type'] == 'none' or (not link_deps and not extra_link_deps):
       # TODO(evan): don't call this function for 'none' target types, as
@@ skipping 625 matching lines @@
   if flavor == 'win':
     configs = [target_dicts[qualified_target]['configurations'][config_name]
                for qualified_target in target_list]
     shared_system_includes = None
     if not generator_flags.get('ninja_use_custom_environment_files', 0):
       shared_system_includes = \
           gyp.msvs_emulation.ExtractSharedMSVSSystemIncludes(
               configs, generator_flags)
     cl_paths = gyp.msvs_emulation.GenerateEnvironmentFiles(
         toplevel_build, generator_flags, shared_system_includes, OpenOutput)
-    for arch, path in cl_paths.iteritems():
+    for arch, path in sorted(cl_paths.iteritems()):
       if clang_cl:
         # If we have selected clang-cl, use that instead.
         path = clang_cl
       command = CommandWithWrapper('CC', wrappers,
           QuoteShellArgument(path, 'win'))
       if clang_cl:
         # Use clang-cl to cross-compile for x86 or x86_64.
         command += (' -m32' if arch == 'x86' else ' -m64')
       master_ninja.variable('cl_' + arch, command)
 
@@ skipping 415 matching lines @@
       non_empty_target_names.add(name)
     else:
       empty_target_names.add(name)
 
   if target_short_names:
     # Write a short name to build this target.  This benefits both the
     # "build chrome" case as well as the gyp tests, which expect to be
     # able to run actions and build libraries by their short name.
     master_ninja.newline()
     master_ninja.comment('Short names for targets.')
-    for short_name in target_short_names:
-      master_ninja.build(short_name, 'phony', [x.FinalOutput() for x in
-                                               target_short_names[short_name]])
+    for short_name in sorted(target_short_names):
+      master_ninja.build(short_name, 'phony', ([x.FinalOutput() for x in

[Nico, 2016/01/13 20:16:54]

why the additional parens?

[Zachary Forman, 2016/01/18 20:44:36]

Corrected.

+                                               target_short_names[short_name]]))
 
   # Write phony targets for any empty targets that weren't written yet. As
   # short names are  not necessarily unique only do this for short names that
   # haven't already been output for another target.
   empty_target_names = empty_target_names - non_empty_target_names
   if empty_target_names:
     master_ninja.newline()
     master_ninja.comment('Empty targets (output for completeness).')
     for name in sorted(empty_target_names):
       master_ninja.build(name, 'phony')
 
   if all_outputs:
     master_ninja.newline()
-    master_ninja.build('all', 'phony', list(all_outputs))
+    master_ninja.build('all', 'phony', sorted(all_outputs))
     master_ninja.default(generator_flags.get('default_target', 'all'))
 
   master_ninja_file.close()
 
 
 def PerformBuild(data, configurations, params):
   options = params['options']
   for config in configurations:
     builddir = os.path.join(options.toplevel_dir, 'out', config)
     arguments = ['ninja', '-C', builddir]
@@ skipping 34 matching lines @@
           arglists.append(
               (target_list, target_dicts, data, params, config_name))
         pool.map(CallGenerateOutputForConfig, arglists)
       except KeyboardInterrupt, e:
         pool.terminate()
         raise e
     else:
       for config_name in config_names:
         GenerateOutputForConfig(target_list, target_dicts, data, params,
                                 config_name)