Integrate ninja build with Visual Studio

pylib/gyp/generator/msvs.py

 # Copyright (c) 2012 Google Inc. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import copy
 import ntpath
 import os
 import posixpath
 import re
 import subprocess
@@ skipping 45 matching lines @@
 
 # The msvs specific sections that hold paths
 generator_additional_path_sections = [
     'msvs_cygwin_dirs',
     'msvs_props',
 ]
 
 
 generator_additional_non_configuration_keys = [
     'msvs_cygwin_dirs',
+    'msvs_ninja_out_dir',
     'msvs_cygwin_shell',
     'msvs_large_pdb',
     'msvs_shard',
 ]
 
 
 # List of precompiled header related keys.
 precomp_keys = [
     'msvs_precompiled_header',
     'msvs_precompiled_source',
@@ skipping 826 matching lines @@
 
   platforms = _GetUniquePlatforms(spec)
   p = MSVSProject.Writer(project.path, version, spec['target_name'],
                          project.guid, platforms)
 
   # Get directory project file is in.
   project_dir = os.path.split(project.path)[0]
   gyp_path = _NormalizedSource(project.build_file)
   relative_path_of_gyp_file = gyp.common.RelativePath(gyp_path, project_dir)
 
-  config_type = _GetMSVSConfigurationType(spec, project.build_file)
+  config_type = _GetMSVSConfigurationType(spec, project.build_file, None)
   for config_name, config in spec['configurations'].iteritems():
     _AddConfigurationToMSVSProject(p, spec, config_type, config_name, config)
 
   # Prepare list of sources and excluded sources.
   gyp_file = os.path.split(project.build_file)[1]
   sources, excluded_sources = _PrepareListOfSources(spec, generator_flags,
                                                     gyp_file)
 
   # Add rules.
   actions_to_add = {}
@@ skipping 56 matching lines @@
   Returns:
     The MSVSUserFile object created.
   """
   (domain, username) = _GetDomainAndUserName()
   vcuser_filename = '.'.join([proj_path, domain, username, 'user'])
   user_file = MSVSUserFile.Writer(vcuser_filename, version,
                                   spec['target_name'])
   return user_file
 
 
-def _GetMSVSConfigurationType(spec, build_file):
+def _GetMSVSConfigurationType(spec, build_file, external_builder):
   """Returns the configuration type for this project.
 
   It's a number defined by Microsoft.  May raise an exception.
 
   Args:
       spec: The target dictionary containing the properties of the target.
       build_file: The path of the gyp file.
+      external_builder: The external builder specified in the generator flags.
   Returns:
       An integer, the configuration type.
   """
+  if external_builder != None:
+      return '10'  # Utility type
   try:
     config_type = {
         'executable': '1',  # .exe
         'shared_library': '2',  # .dll
         'loadable_module': '2',  # .dll
         'static_library': '4',  # .lib
         'none': '10',  # Utility type
         }[spec['type']]
   except KeyError:
     if spec.get('type'):
@@ skipping 639 matching lines @@
   config_platform_overrides = {}
   for config_name, c in spec['configurations'].iteritems():
     config_fullname = _ConfigFullName(config_name, c)
     platform = c.get('msvs_target_platform', _ConfigPlatform(c))
     fixed_config_fullname = '%s|%s' % (
         _ConfigBaseName(config_name, _ConfigPlatform(c)), platform)
     config_platform_overrides[config_fullname] = fixed_config_fullname
   return config_platform_overrides
 
 
-def _CreateProjectObjects(target_list, target_dicts, options, msvs_version):
+def _CreateProjectObjects(target_list, target_dicts, options, msvs_version,
+                          set_dependencies):
   """Create a MSVSProject object for the targets found in target list.
 
   Arguments:
     target_list: the list of targets to generate project objects for.
     target_dicts: the dictionary of specifications.
     options: global generator options.
     msvs_version: the MSVSVersion object.
+    set_dependencies: flag to control whether to set dependencies or not.
   Returns:
     A set of created projects, keyed by target.
   """
   global fixpath_prefix
   # Generate each project.
   projects = {}
   for qualified_target in target_list:
     spec = target_dicts[qualified_target]
     if spec['toolset'] != 'target':
       raise GypError(
@@ skipping 11 matching lines @@
         guid=guid,
         spec=spec,
         build_file=build_file,
         config_platform_overrides=overrides,
         fixpath_prefix=fixpath_prefix)
     # Set project toolset if any (MS build only)
     if msvs_version.UsesVcxproj():
       obj.set_msbuild_toolset(
           _GetMsbuildToolsetOfProject(proj_path, spec, msvs_version))
     projects[qualified_target] = obj
-  # Set all the dependencies
-  for project in projects.values():
-    deps = project.spec.get('dependencies', [])
-    deps = [projects[d] for d in deps]
-    project.set_dependencies(deps)
+
+  if set_dependencies:
+    # Set all the dependencies
+    for project in projects.values():
+      deps = project.spec.get('dependencies', [])
+      deps = [projects[d] for d in deps]
+      project.set_dependencies(deps)
   return projects
 
 
 def CalculateVariables(default_variables, params):
   """Generated variables that require params to be known."""
 
   generator_flags = params.get('generator_flags', {})
 
   # Select project file format version (if unset, default to auto detecting).
   msvs_version = MSVSVersion.SelectVisualStudioVersion(
@@ skipping 62 matching lines @@
         target_list, target_dicts, generator_default_variables)
 
   # Prepare the set of configurations.
   configs = set()
   for qualified_target in target_list:
     spec = target_dicts[qualified_target]
     for config_name, config in spec['configurations'].iteritems():
       configs.add(_ConfigFullName(config_name, config))
   configs = list(configs)
 
-  # Figure out all the projects that will be generated and their guids
+  # Figure out all the projects that will be generated and their guids, but
+  # don't set dependencies if we are using an external builder like ninja
+  set_dependencies = generator_flags.get('msvs_external_builder', None) == None
   project_objects = _CreateProjectObjects(target_list, target_dicts, options,
-                                          msvs_version)
+                                          msvs_version, set_dependencies)
 
   # Generate each project.
   missing_sources = []
   for project in project_objects.values():
     fixpath_prefix = project.fixpath_prefix
     missing_sources.extend(_GenerateProject(project, options, msvs_version,
                                             generator_flags))
   fixpath_prefix = None
 
   for build_file in data:
@@ skipping 621 matching lines @@
   namespace = os.path.splitext(gyp_file_name)[0]
   return [
       ['PropertyGroup', {'Label': 'Globals'},
        ['ProjectGuid', guid],
        ['Keyword', 'Win32Proj'],
        ['RootNamespace', namespace],
       ]
   ]
 
 
-def _GetMSBuildConfigurationDetails(spec, build_file):
+def _GetMSBuildConfigurationDetails(spec, build_file, external_builder):
   properties = {}
   for name, settings in spec['configurations'].iteritems():
-    msbuild_attributes = _GetMSBuildAttributes(spec, settings, build_file)
+    msbuild_attributes = _GetMSBuildAttributes(spec, settings, build_file,
+                                               external_builder)
     condition = _GetConfigurationCondition(name, settings)
     character_set = msbuild_attributes.get('CharacterSet')
     _AddConditionalProperty(properties, condition, 'ConfigurationType',
                             msbuild_attributes['ConfigurationType'])
     if character_set:
       _AddConditionalProperty(properties, condition, 'CharacterSet',
                               character_set)
   return _GetMSBuildPropertyGroup(spec, 'Configuration', properties)
 
 
@@ skipping 46 matching lines @@
           'Condition': "exists('%s')" % user_props,
           'Label': 'LocalAppDataPlatform'
          }
         ]
       ]
       for props_file in props:
         import_group.append(['Import', {'Project':props_file}])
       sheets.append(import_group)
     return sheets
 
-def _ConvertMSVSBuildAttributes(spec, config, build_file):
-  config_type = _GetMSVSConfigurationType(spec, build_file)
+def _ConvertMSVSBuildAttributes(spec, config, build_file, external_builder):
+  config_type = _GetMSVSConfigurationType(spec, build_file, external_builder)
   msvs_attributes = _GetMSVSAttributes(spec, config, config_type)
   msbuild_attributes = {}
   for a in msvs_attributes:
     if a in ['IntermediateDirectory', 'OutputDirectory']:
       directory = MSVSSettings.ConvertVCMacrosToMSBuild(msvs_attributes[a])
       if not directory.endswith('\\'):
         directory += '\\'
       msbuild_attributes[a] = directory
     elif a == 'CharacterSet':
       msbuild_attributes[a] = _ConvertMSVSCharacterSet(msvs_attributes[a])
@@ skipping 18 matching lines @@
   if config_type.isdigit():
     config_type = {
         '1': 'Application',
         '2': 'DynamicLibrary',
         '4': 'StaticLibrary',
         '10': 'Utility'
     }[config_type]
   return config_type
 
 
-def _GetMSBuildAttributes(spec, config, build_file):
+def _GetMSBuildAttributes(spec, config, build_file, external_builder):
   if 'msbuild_configuration_attributes' not in config:
-    msbuild_attributes = _ConvertMSVSBuildAttributes(spec, config, build_file)
-
+    msbuild_attributes = _ConvertMSVSBuildAttributes(spec, config, build_file,
+                                                     external_builder)
   else:
-    config_type = _GetMSVSConfigurationType(spec, build_file)
+    config_type = _GetMSVSConfigurationType(spec, build_file, external_builder)
     config_type = _ConvertMSVSConfigurationType(config_type)
     msbuild_attributes = config.get('msbuild_configuration_attributes', {})
     msbuild_attributes.setdefault('ConfigurationType', config_type)
     output_dir = msbuild_attributes.get('OutputDirectory',
                                       '$(SolutionDir)$(Configuration)')
     msbuild_attributes['OutputDirectory'] = _FixPath(output_dir) + '\\'
     if 'IntermediateDirectory' not in msbuild_attributes:
       intermediate = _FixPath('$(Configuration)') + '\\'
       msbuild_attributes['IntermediateDirectory'] = intermediate
     if 'CharacterSet' in msbuild_attributes:
       msbuild_attributes['CharacterSet'] = _ConvertMSVSCharacterSet(
           msbuild_attributes['CharacterSet'])
   if 'TargetName' not in msbuild_attributes:
     prefix = spec.get('product_prefix', '')
     product_name = spec.get('product_name', '$(ProjectName)')
     target_name = prefix + product_name
     msbuild_attributes['TargetName'] = target_name
 
+  if external_builder == 'ninja':

[scottmg, 2013/04/15 19:37:58]

it feels a bit odd to try to genericize to "external_builder", but then have
ninja-specific details in msvs.py

[Dirk Pranke, 2013/04/16 00:41:04]

Maybe we should at least validate the value of external_builder when we start
the generator, to bail out if it's not equal to 'ninja'? One can imagine wanting
to re-use this code for other builders like make, tup, etc.

+    ninja_out_dir = spec.get('msvs_ninja_out_dir', '.') + '/$(Configuration)'
+    msbuild_attributes['OutputDirectory'] = _FixPath(ninja_out_dir) + '\\'
+
   # Make sure that 'TargetPath' matches 'Lib.OutputFile' or 'Link.OutputFile'
   # (depending on the tool used) to avoid MSB8012 warning.
   msbuild_tool_map = {
       'executable': 'Link',
       'shared_library': 'Link',
       'loadable_module': 'Link',
       'static_library': 'Lib',
   }
   msbuild_tool = msbuild_tool_map.get(spec['type'])
   if msbuild_tool:
     msbuild_settings = config['finalized_msbuild_settings']
     out_file = msbuild_settings[msbuild_tool].get('OutputFile')
     if out_file:
       msbuild_attributes['TargetPath'] = _FixPath(out_file)
 
   return msbuild_attributes
 
 
-def _GetMSBuildConfigurationGlobalProperties(spec, configurations, build_file):
+def _GetMSBuildConfigurationGlobalProperties(spec, configurations, build_file,
+                                             external_builder):
   # TODO(jeanluc) We could optimize out the following and do it only if
   # there are actions.
   # TODO(jeanluc) Handle the equivalent of setting 'CYGWIN=nontsec'.
   new_paths = []
   cygwin_dirs = spec.get('msvs_cygwin_dirs', ['.'])[0]
   if cygwin_dirs:
     cyg_path = '$(MSBuildProjectDirectory)\\%s\\bin\\' % _FixPath(cygwin_dirs)
     new_paths.append(cyg_path)
     # TODO(jeanluc) Change the convention to have both a cygwin_dir and a
     # python_dir.
     python_path = cyg_path.replace('cygwin\\bin', 'python_26')
     new_paths.append(python_path)
     if new_paths:
       new_paths = '$(ExecutablePath);' + ';'.join(new_paths)
 
   properties = {}
   for (name, configuration) in sorted(configurations.iteritems()):
     condition = _GetConfigurationCondition(name, configuration)
-    attributes = _GetMSBuildAttributes(spec, configuration, build_file)
+    attributes = _GetMSBuildAttributes(spec, configuration, build_file,
+                                       external_builder)
     msbuild_settings = configuration['finalized_msbuild_settings']
     _AddConditionalProperty(properties, condition, 'IntDir',
                             attributes['IntermediateDirectory'])
     _AddConditionalProperty(properties, condition, 'OutDir',
                             attributes['OutputDirectory'])
     _AddConditionalProperty(properties, condition, 'TargetName',
                             attributes['TargetName'])
 
+    if external_builder != None:

[Dirk Pranke, 2013/04/16 00:41:04]

Nit: this can probably be just 'if external_builder:'

+      # External builders are Utility projects, so we need to explicitly
+      # specify the target extension, based on the spec's type.
+      ext_map = {
+        'executable': '.exe',
+        'shared_library': '.dll',
+        'loadable_module': '.dll',

[scottmg, 2013/04/15 19:37:58]

this looks a lot like the table in MSVSUtil.py, could that be used instead?

+        'static_library': '.lib',
+      }
+      ext = ext_map.get(spec['type'])
+      if ext:
+        _AddConditionalProperty(properties, condition, 'TargetExt', ext)
+
     if attributes.get('TargetPath'):
       _AddConditionalProperty(properties, condition, 'TargetPath',
                               attributes['TargetPath'])
 
     if new_paths:
       _AddConditionalProperty(properties, condition, 'ExecutablePath',
                               new_paths)
     tool_settings = msbuild_settings.get('', {})
     for name, value in sorted(tool_settings.iteritems()):
       formatted_value = _GetValueFormattedForMSBuild('', name, value)
@@ skipping 323 matching lines @@
   spec = project.spec
   configurations = spec['configurations']
   project_dir, project_file_name = os.path.split(project.path)
   msbuildproj_dir = os.path.dirname(project.path)
   if msbuildproj_dir and not os.path.exists(msbuildproj_dir):
     os.makedirs(msbuildproj_dir)
   # Prepare list of sources and excluded sources.
   gyp_path = _NormalizedSource(project.build_file)
   relative_path_of_gyp_file = gyp.common.RelativePath(gyp_path, project_dir)
 
+  external_builder = generator_flags.get('msvs_external_builder', None)

[Dirk Pranke, 2013/04/16 00:41:04]

Nit: we should verify that external_builder == 'ninja' if it's set at all, and
return some sort of helpful error message otherwise.

+
   gyp_file = os.path.split(project.build_file)[1]
   sources, excluded_sources = _PrepareListOfSources(spec, generator_flags,
                                                     gyp_file)
   # Add rules.
   actions_to_add = {}
   props_files_of_rules = set()
   targets_files_of_rules = set()
   extension_to_rule_name = {}
   list_excluded = generator_flags.get('msvs_list_excluded_files', True)
-  _GenerateRulesForMSBuild(project_dir, options, spec,
-                           sources, excluded_sources,
-                           props_files_of_rules, targets_files_of_rules,
-                           actions_to_add, extension_to_rule_name)
+
+  # Don't generate rules if we are using an external builder like ninja.
+  if external_builder != None:

[Dirk Pranke, 2013/04/16 00:41:04]

same nit.

+    _GenerateRulesForMSBuild(project_dir, options, spec,
+                             sources, excluded_sources,
+                             props_files_of_rules, targets_files_of_rules,
+                             actions_to_add, extension_to_rule_name)
+
   sources, excluded_sources, excluded_idl = (
       _AdjustSourcesAndConvertToFilterHierarchy(spec, options,
                                                 project_dir, sources,
                                                 excluded_sources,
                                                 list_excluded))
-  _AddActions(actions_to_add, spec, project.build_file)
-  _AddCopies(actions_to_add, spec)
 
-  # NOTE: this stanza must appear after all actions have been decided.
-  # Don't excluded sources with actions attached, or they won't run.
-  excluded_sources = _FilterActionsFromExcluded(
-      excluded_sources, actions_to_add)
+  # Don't add actions if we are using an external builder like ninja.
+  if external_builder != None:

[Dirk Pranke, 2013/04/16 00:41:04]

same nit.

+    _AddActions(actions_to_add, spec, project.build_file)
+    _AddCopies(actions_to_add, spec)
+
+    # NOTE: this stanza must appear after all actions have been decided.
+    # Don't excluded sources with actions attached, or they won't run.
+    excluded_sources = _FilterActionsFromExcluded(
+        excluded_sources, actions_to_add)
 
   exclusions = _GetExcludedFilesFromBuild(spec, excluded_sources, excluded_idl)
   actions_spec, sources_handled_by_action = _GenerateActionsForMSBuild(
       spec, actions_to_add)
 
   _GenerateMSBuildFiltersFile(project.path + '.filters', sources,
                               extension_to_rule_name)
   missing_sources = _VerifySourcesExist(sources, project_dir)
 
   for configuration in configurations.itervalues():
@@ skipping 12 matching lines @@
   content = [
       'Project',
       {'xmlns': 'http://schemas.microsoft.com/developer/msbuild/2003',
        'ToolsVersion': version.ProjectVersion(),
        'DefaultTargets': 'Build'
       }]
 
   content += _GetMSBuildProjectConfigurations(configurations)
   content += _GetMSBuildGlobalProperties(spec, project.guid, project_file_name)
   content += import_default_section
-  content += _GetMSBuildConfigurationDetails(spec, project.build_file)
+  content += _GetMSBuildConfigurationDetails(spec, project.build_file,
+                                             external_builder)
   content += _GetMSBuildLocalProperties(project.msbuild_toolset)
   content += import_cpp_props_section
   content += _GetMSBuildExtensions(props_files_of_rules)
   content += _GetMSBuildPropertySheets(configurations)
   content += macro_section
   content += _GetMSBuildConfigurationGlobalProperties(spec, configurations,
-                                                      project.build_file)
+                                                      project.build_file,
+                                                      external_builder)
   content += _GetMSBuildToolSettingsSections(spec, configurations)
   content += _GetMSBuildSources(
       spec, sources, exclusions, extension_to_rule_name, actions_spec,
       sources_handled_by_action, list_excluded)
   content += _GetMSBuildProjectReferences(project)
   content += import_cpp_targets_section
   content += _GetMSBuildExtensionTargets(targets_files_of_rules)
 
+  if external_builder == 'ninja':
+    content += _GetMSBuildNinjaTarget(spec, 'Build',
+                                      ['-j', '${NUMBER_OF_PROCESSORS_PLUS_1}'])

[scottmg, 2013/04/15 19:37:58]

ninja determines -j automatically, don't specify it manually

+    content += _GetMSBuildNinjaTarget(spec, 'Clean', ['-t', 'clean'])
+
   # TODO(jeanluc) File a bug to get rid of runas.  We had in MSVS:
   # has_run_as = _WriteMSVSUserFile(project.path, version, spec)
 
   easy_xml.WriteXmlIfChanged(content, project.path, pretty=True, win32=True)
 
   return missing_sources
 
+def _GetMSBuildNinjaTarget(spec, msbuild_target_name, ninja_args):

[scottmg, 2013/04/15 19:37:58]

docstring

+  cmd = [
+    'ninja.exe',

[scottmg, 2013/04/15 19:37:58]

i don't love the hardcoding here either. perhaps a variable to at least allow a
ninja binary that's not in the path.

+    '-C',
+    '$(OutDir)',
+  ]
+  cmd.extend(ninja_args)
+  cmd.append(spec['target_name'])
+  cmd = _BuildCommandLineForRuleRaw(spec, cmd, True, False, True, True)
+  target = ['Target', {'Name': msbuild_target_name}]
+  target.append(['Exec', {'Command': cmd}])
+  return [target]
 
 def _GetMSBuildExtensions(props_files_of_rules):
   extensions = ['ImportGroup', {'Label': 'ExtensionSettings'}]
   for props_file in props_files_of_rules:
     extensions.append(['Import', {'Project': props_file}])
   return [extensions]
 
 
 def _GetMSBuildExtensionTargets(targets_files_of_rules):
   targets_node = ['ImportGroup', {'Label': 'ExtensionTargets'}]
@@ skipping 66 matching lines @@
   action_spec.extend(
       # TODO(jeanluc) 'Document' for all or just if as_sources?
       [['FileType', 'Document'],
        ['Command', command],
        ['Message', description],
        ['Outputs', outputs]
       ])
   if additional_inputs:
     action_spec.append(['AdditionalInputs', additional_inputs])
   actions_spec.append(action_spec)