Adding cross-compile ability to the make build

pylib/gyp/generator/make.py

 #!/usr/bin/python
 
 # Notes:
 #
 # This is all roughly based on the Makefile system used by the Linux
 # kernel, but is a non-recursive make -- we put the entire dependency
 # graph in front of make and let it figure it out.
 #
 # The code below generates a separate .mk file for each target, but
 # all are sourced by the top-level Makefile.  This means that all
@@ skipping 17 matching lines @@
 import pprint
 
 generator_default_variables = {
   'EXECUTABLE_PREFIX': '',
   'EXECUTABLE_SUFFIX': '',
   'OS': 'linux',
   'STATIC_LIB_PREFIX': 'lib',
   'SHARED_LIB_PREFIX': 'lib',
   'STATIC_LIB_SUFFIX': '.a',
   'SHARED_LIB_SUFFIX': '.so',
-  'INTERMEDIATE_DIR': '$(obj)/geni',
+  'INTERMEDIATE_DIR': '$(obj).$(TOOLSET)/geni',
   'SHARED_INTERMEDIATE_DIR': '$(obj)/gen',
   'PRODUCT_DIR': '$(builddir)',
   'LIB_DIR': '$(obj)',
   'RULE_INPUT_ROOT': '%(INPUT_ROOT)s',  # This gets expanded by Python.
   'RULE_INPUT_PATH': '$(abspath $<)',
 
   # These appear unused -- ???
   'RULE_INPUT_EXT': 'XXXEXT$(suffix $^)',
   'RULE_INPUT_NAME': 'XXXNAME$(notdir $(basename $^)0',
   'CONFIGURATION_NAME': '$(BUILDTYPE)',
 }
 
+# Make supports multiple toolsets
+generator_supports_multiple_toolsets = True
 
 def ensure_directory_exists(path):
   dir = os.path.dirname(path)
   if dir and not os.path.exists(dir):
     os.makedirs(dir)
 
 # Header of toplevel Makefile.
 # This should go into the build tree, but it's easier to keep it here for now.
 SHARED_HEADER = ("""\
 # We borrow heavily from the kernel build setup, though we are simpler since
@@ skipping 23 matching lines @@
 # Object output directory.
 obj := $(builddir)/obj
 abs_obj := $(abspath $(obj))
 
 # We build up a list of all targets so we can slurp in the generated
 # dependency rule Makefiles in one pass.
 all_targets :=
 
 # C++ apps need to be linked with g++.  Not sure what's appropriate.
 LINK := $(CXX)
-RANLIB ?= ranlib
+
+CC.target ?= $(CC)
+CXX.target ?= $(CXX)
+LINK.target ?= $(LINK)
+AR.target ?= $(AR)
+RANLIB.target ?= ranlib
+
+CC.host ?= gcc
+CXX.host ?= g++
+LINK.host ?= g++
+AR.host ?= ar
+RANLIB.host ?= ranlib
 
 # Flags to make gcc output dependency info.  Note that you need to be
 # careful here to use the flags that ccache and distcc can understand.
 # We write to a temporary dep file first and then rename at the end
 # so we can't end up with a broken dep file.
 depfile = $@.d
 DEPFLAGS = -MMD -MF $(depfile).tmp
 
 # We have to fixup the deps output in a few ways.
 # First, the file output should to mention the proper .o file.
@@ skipping 19 matching lines @@
     -e "/./s|$$|:|" $(depfile).tmp >> $(depfile).tmp
 cat $(depfile).tmp >> $(depfile)
 rm -f $(depfile).tmp
 endef
 """
 """
 # Command definitions:
 # - cmd_foo is the actual command to run;
 # - quiet_cmd_foo is the brief-output summary of the command.
 
-quiet_cmd_cc = CC $@
-cmd_cc = $(CC) $(CFLAGS) $(DEPFLAGS) -c -o $@ $<
+quiet_cmd_cc = CC($(TOOLSET)) $@
+cmd_cc = $(CC.$(TOOLSET)) $(CFLAGS) $(DEPFLAGS) -c -o $@ $<
 
-quiet_cmd_cxx = CXX $@
-cmd_cxx = $(CXX) $(CXXFLAGS) $(DEPFLAGS) -c -o $@ $<
+quiet_cmd_cxx = CXX($(TOOLSET)) $@
+cmd_cxx = $(CXX.$(TOOLSET)) $(CXXFLAGS) $(DEPFLAGS) -c -o $@ $<
 
-quiet_cmd_alink = AR+RANLIB $@
-cmd_alink = rm -f $@ && $(AR) rc $@ $(filter %.o,$^) && $(RANLIB) $@
+quiet_cmd_alink = AR+RANLIB($(TOOLSET)) $@
+cmd_alink = rm -f $@ && $(AR.$(TOOLSET)) rc $@ $(filter %.o,$^) && $(RANLIB.$(TOOLSET)) $@
 
 quiet_cmd_touch = TOUCH $@
 cmd_touch = touch $@
 
 quiet_cmd_copy = COPY $@
 cmd_copy = ln -f $< $@ || cp -af $< $@
 
 # Due to circular dependencies between libraries :(, we wrap the
 # special "figure out circular dependencies" flags around the entire
 # input list during linking.
-quiet_cmd_link = LINK $@
-cmd_link = $(LINK) $(LDFLAGS) -o $@ -Wl,--start-group $(filter-out FORCE_DO_CMD, $^) -Wl,--end-group $(LIBS)
+quiet_cmd_link = LINK($(TOOLSET)) $@
+cmd_link = $(LINK.$(TOOLSET)) $(LDFLAGS) -o $@ -Wl,--start-group $(filter-out FORCE_DO_CMD, $^) -Wl,--end-group $(LIBS)
 
 # Shared-object link (for generating .so).
 # Set SONAME to the library filename so our binaries don't reference the local,
 # absolute paths used on the link command-line.
 # TODO: perhaps this can share with the LINK command above?
-quiet_cmd_solink = SOLINK $@
-cmd_solink = $(LINK) -shared $(LDFLAGS) -Wl,-soname=$(@F) -o $@ -Wl,--start-group $(filter-out FORCE_DO_CMD, $^) -Wl,--end-group $(LIBS)
+quiet_cmd_solink = SOLINK($(TOOLSET)) $@
+cmd_solink = $(LINK.$(TOOLSET)) -shared $(LDFLAGS) -Wl,-soname=$(@F) -o $@ -Wl,--start-group $(filter-out FORCE_DO_CMD, $^) -Wl,--end-group $(LIBS)
 """
 r"""
 # Define an escape_quotes function to escape single quotes.
 # This allows us to handle quotes properly as long as we always use
 # use single quotes and escape_quotes.
 escape_quotes = $(subst ','\'',$(1))
 # This comment is here just to include a ' to unconfuse syntax highlighting.
 # Define an escape_vars function to escape '$' variable syntax.
 # This allows us to read/write command lines wth shell variables (e.g.
 # $LD_LIBRARY_PATH), without triggering make substitution.
@@ skipping 36 matching lines @@
 # Declare "all" target first so it is the default, even though we don't have the
 # deps yet.
 .PHONY: all
 all:
 
 # Use FORCE_DO_CMD to force a target to run.  Should be coupled with
 # do_cmd.
 .PHONY: FORCE_DO_CMD
 FORCE_DO_CMD:
 
+""")
+
+SHARED_HEADER_SUFFIX_RULES = ("""\
 # Suffix rules, putting all outputs into $(obj).
-$(obj)/%.o: $(srcdir)/%.c FORCE_DO_CMD
+$(obj).$(TOOLSET)/%.o: $(srcdir)/%.c FORCE_DO_CMD
         @$(call do_cmd,cc,1)
-
-$(obj)/%.o: $(srcdir)/%.s FORCE_DO_CMD
+$(obj).$(TOOLSET)/%.o: $(srcdir)/%.s FORCE_DO_CMD
         @$(call do_cmd,cc)
-
-$(obj)/%.o: $(srcdir)/%.S FORCE_DO_CMD
+$(obj).$(TOOLSET)/%.o: $(srcdir)/%.S FORCE_DO_CMD
         @$(call do_cmd,cc)
-
-$(obj)/%.o: $(srcdir)/%.cpp FORCE_DO_CMD
+$(obj).$(TOOLSET)/%.o: $(srcdir)/%.cpp FORCE_DO_CMD
         @$(call do_cmd,cxx,1)
-$(obj)/%.o: $(srcdir)/%.cc FORCE_DO_CMD
+$(obj).$(TOOLSET)/%.o: $(srcdir)/%.cc FORCE_DO_CMD
         @$(call do_cmd,cxx,1)
-$(obj)/%.o: $(srcdir)/%.cxx FORCE_DO_CMD
+$(obj).$(TOOLSET)/%.o: $(srcdir)/%.cxx FORCE_DO_CMD
         @$(call do_cmd,cxx,1)
 
 # Try building from generated source, too.
-$(obj)/%.o: $(obj)/%.c FORCE_DO_CMD
+$(obj).$(TOOLSET)/%.o: $(obj).$(TOOLSET)/%.c FORCE_DO_CMD
         @$(call do_cmd,cc,1)
-$(obj)/%.o: $(obj)/%.cc FORCE_DO_CMD
+$(obj).$(TOOLSET)/%.o: $(obj).$(TOOLSET)/%.cc FORCE_DO_CMD
         @$(call do_cmd,cxx,1)
-$(obj)/%.o: $(obj)/%.cpp FORCE_DO_CMD
+$(obj).$(TOOLSET)/%.o: $(obj).$(TOOLSET)/%.cpp FORCE_DO_CMD
+        @$(call do_cmd,cxx,1)
+
+$(obj).$(TOOLSET)/%.o: $(obj)/%.c FORCE_DO_CMD
+        @$(call do_cmd,cc,1)
+$(obj).$(TOOLSET)/%.o: $(obj)/%.cc FORCE_DO_CMD
+        @$(call do_cmd,cxx,1)
+$(obj).$(TOOLSET)/%.o: $(obj)/%.cpp FORCE_DO_CMD
         @$(call do_cmd,cxx,1)
 """)
 
 # This gets added to the very beginning of the Makefile.
 SHARED_HEADER_SRCDIR = ("""\
 # The source directory tree.
 srcdir := %s
 
 """)
 
@@ skipping 36 matching lines @@
   """Translate a compilable filename to its .o target."""
   return os.path.splitext(filename)[0] + '.o'
 
 
 def QuoteIfNecessary(string):
   if '"' in string:
     string = '"' + string.replace('"', '\\"') + '"'
   return string
 
 
-def Objectify(path):
-  """Convert a path to its output directory form."""
-  if '$(' in path:
-    return path
-  return '$(obj)/' + path
-
 srcdir_prefix = ''
 def Sourceify(path):
   """Convert a path to its source directory form."""
   if '$(' in path:
     return path
   return srcdir_prefix + path
 
 
 # Map from qualified target to path to output.
 target_outputs = {}
@@ skipping 23 matching lines @@
 
     ensure_directory_exists(output_filename)
 
     self.fp = open(output_filename, 'w')
 
     self.fp.write(header)
 
     self.path = base_path
     self.target = spec['target_name']
     self.type = spec['type']
+    self.toolset = spec['toolset']
 
     deps, link_deps = self.ComputeDeps(spec)
 
     # Some of the generation below can add extra output, sources, or
     # link dependencies.  All of the out params of the functions that
     # follow use names like extra_foo.
     extra_outputs = []
     extra_sources = []
     extra_link_deps = []
 
     self.output = self.ComputeOutput(spec)
     self._INSTALLABLE_TARGETS = ('executable', 'loadable_module',
                                  'shared_library')
     if self.type in self._INSTALLABLE_TARGETS:
       self.alias = os.path.basename(self.output)
     else:
       self.alias = self.output
 
+    self.WriteLn("TOOLSET := " + self.toolset)
+
     # Actions must come first, since they can generate more OBJs for use below.
     if 'actions' in spec:
       self.WriteActions(spec['actions'], extra_sources, extra_outputs)
 
     # Rules must be early like actions.
     if 'rules' in spec:
       self.WriteRules(spec['rules'], extra_sources, extra_outputs)
 
     if 'copies' in spec:
       self.WriteCopies(spec['copies'], extra_outputs)
@@ skipping 207 matching lines @@
       self.WriteLn("# Flags passed to only C (and not C++) files.");
       self.WriteList(config.get('cflags_c'), 'CFLAGS_C_%s' % configname)
       self.WriteLn("# Flags passed to only C++ (and not C) files.");
       self.WriteList(config.get('cflags_cc'), 'CFLAGS_CC_%s' % configname)
       includes = config.get('include_dirs')
       if includes:
         includes = map(Sourceify, map(self.Absolutify, includes))
       self.WriteList(includes, 'INCS_%s' % configname, prefix='-I')
 
     sources = filter(Compilable, sources)
-    objs = map(Objectify, map(self.Absolutify, map(Target, sources)))
+    objs = map(self.Objectify, map(self.Absolutify, map(Target, sources)))
     self.WriteList(objs, 'OBJS')
 
     self.WriteLn('# Add to the list of files we specially track '
                  'dependencies for.')
     self.WriteLn('all_targets += $(OBJS)')
     self.WriteLn()
 
     # Make sure our dependencies are built first.
     if deps:
       self.WriteMakeRule(['$(OBJS)'], deps,
                          comment = 'Make sure our dependencies are built '
                                    'before any of us.',
                          order_only = True)
 
     # Make sure the actions and rules run first.
     # If they generate any extra headers etc., the per-.o file dep tracking
     # will catch the proper rebuilds, so order only is still ok here.
     if extra_outputs:
       self.WriteMakeRule(['$(OBJS)'], extra_outputs,
                          comment = 'Make sure our actions/rules run '
                                    'before any of us.',
                          order_only = True)
 
     if objs:
       extra_link_deps.append('$(OBJS)')
       self.WriteLn("""\
 # CFLAGS et al overrides must be target-local.
 # See "Target-specific Variable Values" in the GNU Make manual.""")
+      self.WriteLn("$(OBJS): TOOLSET := $(TOOLSET)")
       self.WriteLn("$(OBJS): CFLAGS := $(CFLAGS_$(BUILDTYPE)) "
                    "$(CFLAGS_C_$(BUILDTYPE)) "
                    "$(DEFS_$(BUILDTYPE)) $(INCS_$(BUILDTYPE))")
       self.WriteLn("$(OBJS): CXXFLAGS := $(CFLAGS_$(BUILDTYPE)) "
                    "$(CFLAGS_CC_$(BUILDTYPE)) "
                    "$(DEFS_$(BUILDTYPE)) $(INCS_$(BUILDTYPE))")
 
     self.WriteLn()
 
 
@@ skipping 12 matching lines @@
     elif self.type == 'none':
       target = '%s.stamp' % target
     elif self.type == 'settings':
       return None
     elif self.type == 'executable':
       target = spec.get('product_name', target)
     else:
       print ("ERROR: What output file should be generated?",
              "typ", self.type, "target", target)
 
-    path = spec.get('product_dir', os.path.join('$(obj)', self.path))
+    path = spec.get('product_dir', os.path.join('$(obj).' + self.toolset,
+                                                self.path))
     return os.path.join(path, target)
 
 
   def ComputeDeps(self, spec):
     """Compute the dependencies of a gyp spec.
 
     Returns a tuple (deps, link_deps), where each is a list of
     filenames that will need to be put in front of make for either
     building (deps) or linking (link_deps).
     """
@@ skipping 120 matching lines @@
     phony: if true, the rule does not actually generate the named output, the
            output is just a name to run the rule
     """
     if comment:
       self.WriteLn('# ' + comment)
     if phony:
       self.WriteLn('.PHONY: ' + ' '.join(outputs))
     # TODO(evanm): just make order_only a list of deps instead of these hacks.
     order_insert = '| ' if order_only else ''
     force_append = ' FORCE_DO_CMD' if force else ''
+    if actions:
+      self.WriteLn("%s: TOOLSET := $(TOOLSET)" % outputs[0])
     self.WriteLn('%s: %s%s%s' % (outputs[0], order_insert, ' '.join(inputs),
                                  force_append))
     if actions:
       for action in actions:
         self.WriteLn('\t%s' % action)
     if len(outputs) > 1:
       # If we have more than one output, a rule like
       #   foo bar: baz
       # that for *each* output we must run the action, potentially
       # in parallel.  That is not what we're trying to write -- what
       # we want is that we run the action once and it generates all
       # the files.
       # http://www.gnu.org/software/hello/manual/automake/Multiple-Outputs.html
       # discusses this problem and has this solution:
       # 1) Write the naive rule that would produce parallel runs of
       # the action.
       # 2) Make the outputs seralized on each other, so we won't start
       # a a parallel run until the first run finishes, at which point
       # we'll have generated all the outputs and we're done.
       self.WriteLn('%s: %s' % (' '.join(outputs[1:]), outputs[0]))
     self.WriteLn()
 
 
   def WriteLn(self, text=''):
     self.fp.write(text + '\n')
 
 
+  def Objectify(self, path):
+    """Convert a path to its output directory form."""
+    if '$(' in path:
+      path = path.replace('$(obj)/', '$(obj).%s/' % self.toolset)
+      return path
+    return '$(obj).%s/%s' % (self.toolset, path)
+
   def Absolutify(self, path):
     """Convert a subdirectory-relative path into a base-relative path.
     Skips over paths that contain variables."""
     if '$(' in path:
       return path
     return os.path.normpath(os.path.join(self.path, path))
 
 
   def FixupArgPath(self, arg):
     if '/' in arg or '.h.' in arg:
@@ skipping 14 matching lines @@
 
 def GenerateOutput(target_list, target_dicts, data, params):
   options = params['options']
   generator_flags = params.get('generator_flags', {})
   builddir_name = generator_flags.get('output_dir', 'out')
 
   # TODO:  search for the first non-'Default' target.  This can go
   # away when we add verification that all targets have the
   # necessary configurations.
   default_configuration = None
+  toolsets = set([target_dicts[target]['toolset'] for target in target_list])
   for target in target_list:
     spec = target_dicts[target]
     if spec['default_configuration'] != 'Default':
       default_configuration = spec['default_configuration']
       break
   if not default_configuration:
     default_configuration = 'Default'
 
   srcdir = '.'
   makefile_name = 'Makefile' + options.suffix
   makefile_path = os.path.join(options.depth, makefile_name)
   if options.generator_output:
     global srcdir_prefix
     makefile_path = os.path.join(options.generator_output, makefile_path)
     srcdir = gyp.common.RelativePath('.', options.generator_output)
     srcdir_prefix = '$(srcdir)/'
   ensure_directory_exists(makefile_path)
   root_makefile = open(makefile_path, 'w')
   root_makefile.write(SHARED_HEADER_SRCDIR % srcdir)
   root_makefile.write(SHARED_HEADER_BUILDDIR_NAME % builddir_name)
   root_makefile.write(SHARED_HEADER.replace('__default_configuration__',
                                             default_configuration))
+  for toolset in toolsets:
+    root_makefile.write('TOOLSET := %s\n' % toolset)
+    root_makefile.write(SHARED_HEADER_SUFFIX_RULES)
 
   # Find the list of targets that derive from the gyp file(s) being built.
   needed_targets = set()
   for build_file in params['build_files']:
     for target in gyp.common.AllTargets(target_list, target_dicts, build_file):
       needed_targets.add(target)
 
   build_files = set()
   for qualified_target in target_list:
     # target_list is all the targets defined in all referenced gyp files, so
     # weed out the ones that aren't actually needed for this build. We could
     # generate the .mk even for unneeded targets, as long as we don't
     # reference them in the top-level Makefile, but skipping them entirely
     # should work too.
     if qualified_target not in needed_targets:
       continue
-    build_file, target = gyp.common.BuildFileAndTarget('', qualified_target)[:2]
+    build_file, target, toolset = gyp.common.ParseQualifiedTarget(
+        qualified_target)
     build_files.add(gyp.common.RelativePath(build_file, options.depth))
     included_files = data[build_file]['included_files']
     for included_file in included_files:
       # The included_files entries are relative to the dir of the build file
       # that included them, so we have to undo that and then make them relative
       # to the root dir.
       build_files.add(gyp.common.RelativePath(
           gyp.common.UnrelativePath(included_file, build_file), options.depth))
 
     # Paths in gyp files are relative to the .gyp file, but we want
     # paths relative to the source root for the master makefile.  Grab
     # the path of the .gyp file as the base to relativize against.
     # E.g. "foo/bar" when we're constructing targets found "foo/bar/baz.gyp".
     base_path = gyp.common.RelativePath(os.path.dirname(build_file),
                                         options.depth)
     # We write the .mk file in the base_path directory.
     output_file = os.path.join(options.depth,
                                base_path,
-                               target + options.suffix + '.mk')
+                               target + '.' + toolset + options.suffix + '.mk')
 
     if options.generator_output:
       output_file = os.path.join(options.generator_output, output_file)
 
     spec = target_dicts[qualified_target]
     configs = spec['configurations']
 
     writer = MakefileWriter()
     writer.Write(qualified_target, base_path, output_file, spec, configs)
 
@@ skipping 11 matching lines @@
         ' '.join(map(Sourceify, build_files)),
         gyp.common.EncodePOSIXShellList(
             [gyp.common.FixIfRelativePath(params['gyp_binary'], options.depth),
              '-fmake'] +
             gyp.RegenerateFlags(options) +
             build_files_args)))
 
   root_makefile.write(SHARED_FOOTER)
 
   root_makefile.close()