Create msvs_large_pdb workaround.

pylib/gyp/MSVSUtil.py

 # Copyright (c) 2013 Google Inc. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Utility functions shared amongst the Windows generators."""
 
 import copy
+import os
+
+
+_THIS_DIR = os.path.abspath(os.path.dirname(__file__))
+_SRC_DIR = os.path.abspath(os.path.join(_THIS_DIR, '..', '..'))
+_WIN_DATA_DIR = os.path.join(_SRC_DIR, 'data', 'win')
+_LARGE_PDB_SHIM_CC = os.path.join(_WIN_DATA_DIR, 'large-pdb-shim.cc')
+

[Evan Martin, 2013/03/06 20:43:39]

Nit: rather than put all of these variables at module scope, might be better to
def _GetPathToLargePDBShim() that contains this logic.

[chrisha, 2013/03/14 15:57:23]

Done.

+
+_TARGET_TYPE_EXT = {
+  'executable': '.exe',
+  'shared_library': '.dll'
+}
+
+
+def _SuffixName(name, suffix):
+  """Add a suffix to the end of a target.
+
+  Arguments:
+    name: name of the target (foo#target)
+    suffix: the suffix to be added
+  Returns:
+    Target name with suffix added (foo_suffix#target)
+  """
+  parts = name.rsplit('#', 1)
+  parts[0] = '%s_%s' % (parts[0], suffix)
+  return '#'.join(parts)
+
 
 def _ShardName(name, number):
   """Add a shard number to the end of a target.
 
   Arguments:
     name: name of the target (foo#target)
     number: shard number
   Returns:
     Target name with shard added (foo_1#target)
   """
-  parts = name.rsplit('#', 1)
-  parts[0] = '%s_%d' % (parts[0], number)
-  return '#'.join(parts)
+  return _SuffixName(name, str(number))
 
 
 def ShardTargets(target_list, target_dicts):
   """Shard some targets apart to work around the linkers limits.
 
   Arguments:
     target_list: List of target pairs: 'base/base.gyp:base'.
     target_dicts: Dict of target properties keyed on target pair.
   Returns:
     Tuple of the new sharded versions of the inputs.
@@ skipping 34 matching lines @@
     new_dependencies = []
     for d in dependencies:
       if d in targets_to_shard:
         for i in range(targets_to_shard[d]):
           new_dependencies.append(_ShardName(d, i))
       else:
         new_dependencies.append(d)
     new_target_dicts[t]['dependencies'] = new_dependencies
 
   return (new_target_list, new_target_dicts)
+
+
+def InsertLargePdbShims(target_list, target_dicts, vars):
+  """Insert a shim target that forces the linker to use 4KB pagesize PDBs.
+
+  This is a workaround for targets with PDBs greater than 1GB in size, the
+  limit for the 1KB pagesize PDBs created by the linker by default.
+
+  Arguments:
+    target_list: List of target pairs: 'base/base.gyp:base'.
+    target_dicts: Dict of target properties keyed on target pair.
+    vars: A dictionary of common GYP variables with generator-specific values.
+  Returns:
+    Tuple of the shimmed version of the inputs.
+  """
+  # Determine which targets need shimming.
+  targets_to_shim = []
+  for t in target_dicts:
+    d = target_dicts[t]
+    # We only want to shim targets that have msvs_large_pdb enabled.
+    if d.get('msvs_large_pdb', '0') != '1':
+      continue
+    # This is only intended for use by linker targets that are producing a
+    # PE file.
+    if d['type'] not in ['executable', 'shared_library']:

[scottmg, 2013/03/06 19:03:34]

is all of this checking necessary? can't we just put msvs_large_pdb only on
things that actually need it?

[Evan Martin, 2013/03/06 20:43:39]

I think it's better to fail (or at least warn) loudly than silently ignore it,
too.

[chrisha, 2013/03/14 15:57:23]

Done.

+      continue
+    # This is only intended for use with targets that produce PDB output in
+    # all configurations.
+    has_pdb = True
+    for config in d['configurations'].itervalues():
+      linker = config.get('msvs_settings', {}).get('VCLinkerTool', {})
+      if linker.get('GenerateDebugInformation', 'false') != 'true':
+        has_pdb = False
+        continue
+      if len(linker.get('ProgramDatabaseFile', '')) == 0:
+        has_pdb = False
+        continue
+    if not has_pdb:
+      continue
+
+    # This target is eligible for shimming.
+    targets_to_shim.append(t)
+
+  for t in targets_to_shim:
+    d = target_dicts[t]
+    target_name = d.get('target_name')
+
+    # This is the dict for copying the source file (part of the GYP tree)
+    # to the intermediate directory of the project. This is necessary otherwise
+    # we can't always build a relative path to the shim source file, and Ninja

[scottmg, 2013/03/06 19:03:34]

sorry, why can't we build a relative path?

if it's not possible, since the file doesn't need anything in it, i think it'd
be preferable to just output an empty .cc in the intermediate dir at generation
time.

[chrisha, 2013/03/14 15:57:23]

Because on windows your gyp install might be on a different drive from the
project in question, unfortunately. I'd prefer to simply create the file at
generation time, but are we guaranteed that the output directories exist? This
bit of code seems to run before that is the case.

+    # hates absolute paths (it ends up generating the .obj and .obj.d alongside
+    # the source file, polluting GYPs tree).
+    copy_suffix = '_large_pdb_copy'
+    copy_target_name = target_name + '_' + copy_suffix
+    full_copy_target_name = _SuffixName(t, copy_suffix)
+    shim_cc_basename = os.path.basename(_LARGE_PDB_SHIM_CC)
+    shim_cc_dir = vars['SHARED_INTERMEDIATE_DIR'] + '/' + copy_target_name
+    shim_cc_path = shim_cc_dir + '/' + shim_cc_basename
+    copy_dict = copy.deepcopy(d)
+    copy_dict['target_name'] = copy_target_name
+    copy_dict['type'] = 'none'
+    copy_dict['sources'] = [ _LARGE_PDB_SHIM_CC ]
+    copy_dict['dependencies'] = []
+    copy_dict['copies'] = [{
+      'destination': shim_cc_dir,
+      'files': [ _LARGE_PDB_SHIM_CC ]
+    }]
+
+    # This is the dict for the PDB generating shim target. It depends on the
+    # copy target.
+    shim_suffix = '_large_pdb_shim'
+    shim_target_name = target_name + '_' + shim_suffix
+    full_shim_target_name = _SuffixName(t, shim_suffix)
+    shim_dict = copy.deepcopy(d)
+    shim_dict['target_name'] = shim_target_name
+    shim_dict['type'] = 'static_library'
+    shim_dict['sources'] = [ shim_cc_path ]
+    shim_dict['dependencies'] = [ full_copy_target_name ]
+    shim_dict['libraries'] = []

[chrisha, 2013/03/05 20:52:25]

I'd love comments as to the sanity of the two deepcopies here. Is there a
description somewhere of what properties are the minimum that need to be set?
Are there are other optional properties I'm not resetting that could cause
problems?

(Excuse my non-familiarity with GYP's internals.)

[scottmg, 2013/03/06 19:03:34]

i'm afraid i don't know the answer to this question. this looks a bit scary
though... i'd be inclined to try to model it after what's in input.py rather
than deepcopy'ing.

[chrisha, 2013/03/14 15:57:23]

Done.

[chrisha, 2013/03/14 15:57:23]

Went with a mix of the two. deepcopied what is absolutely required, injected
things I need. Feels safer ;)

+
+    # Set up the shim to output its PDB to the same location as the final linker
+    # target.
+    for config in shim_dict.get('configurations').itervalues():
+      msvs = config.setdefault('msvs_settings')
+
+      linker = msvs.pop('VCLinkerTool')  # We want to clear this dict.
+      pdb_path = linker.get('ProgramDatabaseFile')
+
+      compiler = msvs.setdefault('VCCLCompilerTool', {})
+      compiler.setdefault('DebugInformationFormat', '3')
+      compiler.setdefault('ProgramDataBaseFileName', pdb_path)
+
+    # Add the new targets.
+    target_list.append(full_copy_target_name)
+    target_list.append(full_shim_target_name)
+    target_dicts[full_copy_target_name] = copy_dict
+    target_dicts[full_shim_target_name] = shim_dict
+
+    # Update the original target to depend on the shim target.
+    d.setdefault('dependencies', []).append(full_shim_target_name)
+
+  return (target_list, target_dicts)