Integrate ninja build with Visual Studio

pylib/gyp/generator/msvs.py

Index: pylib/gyp/generator/msvs.py
diff --git a/pylib/gyp/generator/msvs.py b/pylib/gyp/generator/msvs.py
index 51acf2eb3e8e00901a58ba8dfef436660aa622a7..850cafe7fc73194b1a3640942c65073efeef426e 100644
--- a/pylib/gyp/generator/msvs.py
+++ b/pylib/gyp/generator/msvs.py
@@ -63,6 +63,7 @@ generator_additional_path_sections = [
 
 generator_additional_non_configuration_keys = [
     'msvs_cygwin_dirs',
+    'msvs_ninja_out_dir',
     'msvs_cygwin_shell',
     'msvs_large_pdb',
     'msvs_shard',
@@ -909,7 +910,7 @@ def _GenerateMSVSProject(project, options, version, generator_flags):
   gyp_path = _NormalizedSource(project.build_file)
   relative_path_of_gyp_file = gyp.common.RelativePath(gyp_path, project_dir)
 
-  config_type = _GetMSVSConfigurationType(spec, project.build_file)
+  config_type = _GetMSVSConfigurationType(spec, project.build_file, None)
   for config_name, config in spec['configurations'].iteritems():
     _AddConfigurationToMSVSProject(p, spec, config_type, config_name, config)
 
@@ -986,7 +987,7 @@ def _CreateMSVSUserFile(proj_path, version, spec):
   return user_file
 
 
-def _GetMSVSConfigurationType(spec, build_file):
+def _GetMSVSConfigurationType(spec, build_file, external_builder):
   """Returns the configuration type for this project.
 
   It's a number defined by Microsoft.  May raise an exception.
@@ -994,9 +995,12 @@ def _GetMSVSConfigurationType(spec, build_file):
   Args:
       spec: The target dictionary containing the properties of the target.
       build_file: The path of the gyp file.
+      external_builder: The external builder specified in the generator flags.
   Returns:
       An integer, the configuration type.
   """
+  if external_builder != None:
+      return '10'  # Utility type
   try:
     config_type = {
         'executable': '1',  # .exe
@@ -1656,7 +1660,8 @@ def _GetPlatformOverridesOfProject(spec):
   return config_platform_overrides
 
 
-def _CreateProjectObjects(target_list, target_dicts, options, msvs_version):
+def _CreateProjectObjects(target_list, target_dicts, options, msvs_version,
+                          set_dependencies):
   """Create a MSVSProject object for the targets found in target list.
 
   Arguments:
@@ -1664,6 +1669,7 @@ def _CreateProjectObjects(target_list, target_dicts, options, msvs_version):
     target_dicts: the dictionary of specifications.
     options: global generator options.
     msvs_version: the MSVSVersion object.
+    set_dependencies: flag to control whether to set dependencies or not.
   Returns:
     A set of created projects, keyed by target.
   """
@@ -1695,11 +1701,13 @@ def _CreateProjectObjects(target_list, target_dicts, options, msvs_version):
       obj.set_msbuild_toolset(
           _GetMsbuildToolsetOfProject(proj_path, spec, msvs_version))
     projects[qualified_target] = obj
-  # Set all the dependencies
-  for project in projects.values():
-    deps = project.spec.get('dependencies', [])
-    deps = [projects[d] for d in deps]
-    project.set_dependencies(deps)
+
+  if set_dependencies:
+    # Set all the dependencies
+    for project in projects.values():
+      deps = project.spec.get('dependencies', [])
+      deps = [projects[d] for d in deps]
+      project.set_dependencies(deps)
   return projects
 
 
@@ -1782,9 +1790,11 @@ def GenerateOutput(target_list, target_dicts, data, params):
       configs.add(_ConfigFullName(config_name, config))
   configs = list(configs)
 
-  # Figure out all the projects that will be generated and their guids
+  # Figure out all the projects that will be generated and their guids, but
+  # don't set dependencies if we are using an external builder like ninja
+  set_dependencies = generator_flags.get('msvs_external_builder', None) == None
   project_objects = _CreateProjectObjects(target_list, target_dicts, options,
-                                          msvs_version)
+                                          msvs_version, set_dependencies)
 
   # Generate each project.
   missing_sources = []
@@ -2426,10 +2436,11 @@ def _GetMSBuildGlobalProperties(spec, guid, gyp_file_name):
   ]
 
 
-def _GetMSBuildConfigurationDetails(spec, build_file):
+def _GetMSBuildConfigurationDetails(spec, build_file, external_builder):
   properties = {}
   for name, settings in spec['configurations'].iteritems():
-    msbuild_attributes = _GetMSBuildAttributes(spec, settings, build_file)
+    msbuild_attributes = _GetMSBuildAttributes(spec, settings, build_file,
+                                               external_builder)
     condition = _GetConfigurationCondition(name, settings)
     character_set = msbuild_attributes.get('CharacterSet')
     _AddConditionalProperty(properties, condition, 'ConfigurationType',
@@ -2496,8 +2507,8 @@ def _GetMSBuildPropertySheets(configurations):
       sheets.append(import_group)
     return sheets
 
-def _ConvertMSVSBuildAttributes(spec, config, build_file):
-  config_type = _GetMSVSConfigurationType(spec, build_file)
+def _ConvertMSVSBuildAttributes(spec, config, build_file, external_builder):
+  config_type = _GetMSVSConfigurationType(spec, build_file, external_builder)
   msvs_attributes = _GetMSVSAttributes(spec, config, config_type)
   msbuild_attributes = {}
   for a in msvs_attributes:
@@ -2536,12 +2547,12 @@ def _ConvertMSVSConfigurationType(config_type):
   return config_type
 
 
-def _GetMSBuildAttributes(spec, config, build_file):
+def _GetMSBuildAttributes(spec, config, build_file, external_builder):
   if 'msbuild_configuration_attributes' not in config:
-    msbuild_attributes = _ConvertMSVSBuildAttributes(spec, config, build_file)
-
+    msbuild_attributes = _ConvertMSVSBuildAttributes(spec, config, build_file,
+                                                     external_builder)
   else:
-    config_type = _GetMSVSConfigurationType(spec, build_file)
+    config_type = _GetMSVSConfigurationType(spec, build_file, external_builder)
     config_type = _ConvertMSVSConfigurationType(config_type)
     msbuild_attributes = config.get('msbuild_configuration_attributes', {})
     msbuild_attributes.setdefault('ConfigurationType', config_type)
@@ -2560,6 +2571,10 @@ def _GetMSBuildAttributes(spec, config, build_file):
     target_name = prefix + product_name
     msbuild_attributes['TargetName'] = target_name
 
+  if external_builder == 'ninja':

[scottmg, 2013/04/15 19:37:58]

it feels a bit odd to try to genericize to "external_builder", but then have
ninja-specific details in msvs.py

[Dirk Pranke, 2013/04/16 00:41:04]

Maybe we should at least validate the value of external_builder when we start
the generator, to bail out if it's not equal to 'ninja'? One can imagine wanting
to re-use this code for other builders like make, tup, etc.

+    ninja_out_dir = spec.get('msvs_ninja_out_dir', '.') + '/$(Configuration)'
+    msbuild_attributes['OutputDirectory'] = _FixPath(ninja_out_dir) + '\\'
+
   # Make sure that 'TargetPath' matches 'Lib.OutputFile' or 'Link.OutputFile'
   # (depending on the tool used) to avoid MSB8012 warning.
   msbuild_tool_map = {
@@ -2578,7 +2593,8 @@ def _GetMSBuildAttributes(spec, config, build_file):
   return msbuild_attributes
 
 
-def _GetMSBuildConfigurationGlobalProperties(spec, configurations, build_file):
+def _GetMSBuildConfigurationGlobalProperties(spec, configurations, build_file,
+                                             external_builder):
   # TODO(jeanluc) We could optimize out the following and do it only if
   # there are actions.
   # TODO(jeanluc) Handle the equivalent of setting 'CYGWIN=nontsec'.
@@ -2597,7 +2613,8 @@ def _GetMSBuildConfigurationGlobalProperties(spec, configurations, build_file):
   properties = {}
   for (name, configuration) in sorted(configurations.iteritems()):
     condition = _GetConfigurationCondition(name, configuration)
-    attributes = _GetMSBuildAttributes(spec, configuration, build_file)
+    attributes = _GetMSBuildAttributes(spec, configuration, build_file,
+                                       external_builder)
     msbuild_settings = configuration['finalized_msbuild_settings']
     _AddConditionalProperty(properties, condition, 'IntDir',
                             attributes['IntermediateDirectory'])
@@ -2606,6 +2623,19 @@ def _GetMSBuildConfigurationGlobalProperties(spec, configurations, build_file):
     _AddConditionalProperty(properties, condition, 'TargetName',
                             attributes['TargetName'])
 
+    if external_builder != None:

[Dirk Pranke, 2013/04/16 00:41:04]

Nit: this can probably be just 'if external_builder:'

+      # External builders are Utility projects, so we need to explicitly
+      # specify the target extension, based on the spec's type.
+      ext_map = {
+        'executable': '.exe',
+        'shared_library': '.dll',
+        'loadable_module': '.dll',

[scottmg, 2013/04/15 19:37:58]

this looks a lot like the table in MSVSUtil.py, could that be used instead?

+        'static_library': '.lib',
+      }
+      ext = ext_map.get(spec['type'])
+      if ext:
+        _AddConditionalProperty(properties, condition, 'TargetExt', ext)
+
     if attributes.get('TargetPath'):
       _AddConditionalProperty(properties, condition, 'TargetPath',
                               attributes['TargetPath'])
@@ -2949,6 +2979,8 @@ def _GenerateMSBuildProject(project, options, version, generator_flags):
   gyp_path = _NormalizedSource(project.build_file)
   relative_path_of_gyp_file = gyp.common.RelativePath(gyp_path, project_dir)
 
+  external_builder = generator_flags.get('msvs_external_builder', None)

[Dirk Pranke, 2013/04/16 00:41:04]

Nit: we should verify that external_builder == 'ninja' if it's set at all, and
return some sort of helpful error message otherwise.

+
   gyp_file = os.path.split(project.build_file)[1]
   sources, excluded_sources = _PrepareListOfSources(spec, generator_flags,
                                                     gyp_file)
@@ -2958,22 +2990,29 @@ def _GenerateMSBuildProject(project, options, version, generator_flags):
   targets_files_of_rules = set()
   extension_to_rule_name = {}
   list_excluded = generator_flags.get('msvs_list_excluded_files', True)
-  _GenerateRulesForMSBuild(project_dir, options, spec,
-                           sources, excluded_sources,
-                           props_files_of_rules, targets_files_of_rules,
-                           actions_to_add, extension_to_rule_name)
+
+  # Don't generate rules if we are using an external builder like ninja.
+  if external_builder != None:

[Dirk Pranke, 2013/04/16 00:41:04]

same nit.

+    _GenerateRulesForMSBuild(project_dir, options, spec,
+                             sources, excluded_sources,
+                             props_files_of_rules, targets_files_of_rules,
+                             actions_to_add, extension_to_rule_name)
+
   sources, excluded_sources, excluded_idl = (
       _AdjustSourcesAndConvertToFilterHierarchy(spec, options,
                                                 project_dir, sources,
                                                 excluded_sources,
                                                 list_excluded))
-  _AddActions(actions_to_add, spec, project.build_file)
-  _AddCopies(actions_to_add, spec)
 
-  # NOTE: this stanza must appear after all actions have been decided.
-  # Don't excluded sources with actions attached, or they won't run.
-  excluded_sources = _FilterActionsFromExcluded(
-      excluded_sources, actions_to_add)
+  # Don't add actions if we are using an external builder like ninja.
+  if external_builder != None:

[Dirk Pranke, 2013/04/16 00:41:04]

same nit.

+    _AddActions(actions_to_add, spec, project.build_file)
+    _AddCopies(actions_to_add, spec)
+
+    # NOTE: this stanza must appear after all actions have been decided.
+    # Don't excluded sources with actions attached, or they won't run.
+    excluded_sources = _FilterActionsFromExcluded(
+        excluded_sources, actions_to_add)
 
   exclusions = _GetExcludedFilesFromBuild(spec, excluded_sources, excluded_idl)
   actions_spec, sources_handled_by_action = _GenerateActionsForMSBuild(
@@ -3006,14 +3045,16 @@ def _GenerateMSBuildProject(project, options, version, generator_flags):
   content += _GetMSBuildProjectConfigurations(configurations)
   content += _GetMSBuildGlobalProperties(spec, project.guid, project_file_name)
   content += import_default_section
-  content += _GetMSBuildConfigurationDetails(spec, project.build_file)
+  content += _GetMSBuildConfigurationDetails(spec, project.build_file,
+                                             external_builder)
   content += _GetMSBuildLocalProperties(project.msbuild_toolset)
   content += import_cpp_props_section
   content += _GetMSBuildExtensions(props_files_of_rules)
   content += _GetMSBuildPropertySheets(configurations)
   content += macro_section
   content += _GetMSBuildConfigurationGlobalProperties(spec, configurations,
-                                                      project.build_file)
+                                                      project.build_file,
+                                                      external_builder)
   content += _GetMSBuildToolSettingsSections(spec, configurations)
   content += _GetMSBuildSources(
       spec, sources, exclusions, extension_to_rule_name, actions_spec,
@@ -3022,6 +3063,11 @@ def _GenerateMSBuildProject(project, options, version, generator_flags):
   content += import_cpp_targets_section
   content += _GetMSBuildExtensionTargets(targets_files_of_rules)
 
+  if external_builder == 'ninja':
+    content += _GetMSBuildNinjaTarget(spec, 'Build',
+                                      ['-j', '${NUMBER_OF_PROCESSORS_PLUS_1}'])

[scottmg, 2013/04/15 19:37:58]

ninja determines -j automatically, don't specify it manually

+    content += _GetMSBuildNinjaTarget(spec, 'Clean', ['-t', 'clean'])
+
   # TODO(jeanluc) File a bug to get rid of runas.  We had in MSVS:
   # has_run_as = _WriteMSVSUserFile(project.path, version, spec)
 
@@ -3029,6 +3075,18 @@ def _GenerateMSBuildProject(project, options, version, generator_flags):
 
   return missing_sources
 
+def _GetMSBuildNinjaTarget(spec, msbuild_target_name, ninja_args):

[scottmg, 2013/04/15 19:37:58]

docstring

+  cmd = [
+    'ninja.exe',

[scottmg, 2013/04/15 19:37:58]

i don't love the hardcoding here either. perhaps a variable to at least allow a
ninja binary that's not in the path.

+    '-C',
+    '$(OutDir)',
+  ]
+  cmd.extend(ninja_args)
+  cmd.append(spec['target_name'])
+  cmd = _BuildCommandLineForRuleRaw(spec, cmd, True, False, True, True)
+  target = ['Target', {'Name': msbuild_target_name}]
+  target.append(['Exec', {'Command': cmd}])
+  return [target]
 
 def _GetMSBuildExtensions(props_files_of_rules):
   extensions = ['ImportGroup', {'Label': 'ExtensionSettings'}]