split_link tool, config, and scripts for windows build

tools/win/split_link/split_link.py

+# Copyright 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Takes the same arguments as Windows link.exe, and a definition of libraries
+to split into subcomponents. Does multiple passes of link.exe invocation to
+determine exports between parts and generates .def and import libraries to
+cause symbols to be available to other parts."""
+
+from ctypes import *

[M-A Ruel, 2013/05/14 00:39:09]

Could you enumerate instead? It contains a ton of things.

[scottmg, 2013/05/14 03:35:07]

Done.

+import os
+import re
+import subprocess
+import sys
+
+
+def Log(message):
+  print 'split_link:', message
+
+
+def GetFlagsAndInputs(argv):
+  """Parse the command line intended for link.exe and return the flags and
+  input files."""
+  rsp_expanded = []
+  for arg in argv:
+    if arg[0] == '@':
+      with open(arg[1:]) as rsp:
+        rsp_expanded.extend(rsp.read().splitlines())
+    else:
+      rsp_expanded.append(arg)
+
+  # Use CommandLineToArgvW so we match link.exe parsing.
+  try:
+    size = c_int()
+    ptr = windll.shell32.CommandLineToArgvW(
+        create_unicode_buffer(' '.join(rsp_expanded)),
+        byref(size))
+    ref = c_wchar_p * size.value
+    raw = ref.from_address(ptr)
+    args = [arg for arg in raw]
+  finally:
+    windll.kernel32.LocalFree(ptr)
+
+  inputs = []
+  flags = []
+  for arg in args:
+    lower_arg = arg.lower()
+    # We'll be replacing this ourselves.
+    if lower_arg.startswith('/out:'):
+      continue
+    if (not lower_arg.startswith('/') and
+        lower_arg.endswith(('.obj', '.lib', '.res'))):
+      inputs.append(arg)
+    else:
+      flags.append(arg)
+
+  return flags, inputs
+
+
+def GetOriginalLinkerPath():
+  _winreg = None
+  if sys.platform == 'win32':
+    import _winreg
+  elif sys.platform == 'cygwin':
+    import cygwinreg as _winreg
+
+  try:
+    val = _winreg.QueryValue(_winreg.HKEY_CURRENT_USER,
+                             'Software\\Chromium\\split_link_installed')
+    if os.path.exists(val):
+      return val
+  except WindowsError:
+    pass
+
+  raise SystemExit("Couldn't read linker location from registry")
+
+
+def PartFor(input_file, description_parts, description_all):
+  """Determine which part a given link input should be put into (or all)."""
+  # Check if it should go in all parts.
+  input_file = input_file.lower()
+  if any(re.search(spec, input_file) for spec in description_all):
+    return -1
+  # Or pick which particular one it belongs in.
+  for i, spec_list in enumerate(description_parts):
+    if any(re.search(spec, input_file) for spec in spec_list):
+      return i
+  raise ValueError("couldn't find location for %s" % input_file)
+
+
+def ParseOutExternals(output):
+  """Given the stdout of link.exe, parse the errors messages to retrieve all
+  symbols that are unresolved."""
+  result = set()
+  # Styles of messages for unresolved externals, and a boolean to indicate
+  # whether the error message emits the symbols with or without a leading
+  # underscore.
+  unresolved_regexes = [
+    (re.compile(r' : error LNK2019: unresolved external symbol ".*" \((.*)\)'
+                r' referenced in function'),
+     False),
+    (re.compile(r' : error LNK2001: unresolved external symbol ".*" \((.*)\)$'),
+     False),
+    (re.compile(r' : error LNK2019: unresolved external symbol (.*)'
+                r' referenced in function '),
+     True),
+    (re.compile(r' : error LNK2001: unresolved external symbol (.*)$'),
+     True),
+  ]
+  for line in output.splitlines():
+    line = line.strip()
+    for i, (regex, strip_leading_underscore) in enumerate(unresolved_regexes):
+      mo = regex.search(line)
+      if mo:
+        if strip_leading_underscore:
+          result.add(mo.group(1)[1:])
+        else:
+          result.add(mo.group(1))
+        break
+
+  mo = re.search(r'fatal error LNK1120: (\d+) unresolved externals', output)
+  # Make sure we have the same number that the linker thinks we have.
+  assert mo or not result
+  if len(result) != int(mo.group(1)):
+    print output
+    print 'Expecting %d, got %d' % (int(mo.group(1)), len(result))
+  assert len(result) == int(mo.group(1))
+  return sorted(result)
+
+
+def AsCommandLineArgs(list):
+  """Intended for output to a response file. Quotes all arguments."""
+  return '\n'.join('"' + x + '"' for x in list)
+
+
+def RunLinker(flags, index, inputs, phase):
+  """Invoke the linker and return the stdout, returncode and target name."""
+  rspfile = 'part%d_%s.rsp' % (index, phase)
+  import os

[M-A Ruel, 2013/05/14 00:39:09]

?

[scottmg, 2013/05/14 03:35:07]

No idea. Done.

+  with open(rspfile, 'w') as f:
+    print >>f, AsCommandLineArgs(inputs)
+    print >>f, AsCommandLineArgs(flags)
+    output_name = 'chrome%d.dll' % index
+    print >>f, '/ENTRY:ChromeEmptyEntry@12'
+    print >>f, '/OUT:' + output_name
+  # Log('[[[\n' + open(rspfile).read() + '\n]]]')
+  link_exe = GetOriginalLinkerPath()
+  popen = subprocess.Popen([link_exe, '@' + rspfile], stdout=subprocess.PIPE)
+  stdout, _ = popen.communicate()
+  return stdout, popen.returncode, output_name
+
+
+def GenerateDefFiles(unresolved_by_part):
+  """Given a list of unresolved externals, generate a .def file that will
+  cause all those symbols to be exported."""
+  deffiles = []
+  Log('generating .def files')
+  for i, part in enumerate(unresolved_by_part):
+    deffile = 'part%d.def' % i
+    with open(deffile, 'w') as f:
+      print >>f, 'LIBRARY chrome%d.dll' % i
+      print >>f, 'EXPORTS'
+      for j, part in enumerate(unresolved_by_part):
+        if i == j:
+          continue
+        print >>f, '\n'.join('  ' + export for export in part)
+    deffiles.append(deffile)
+  return deffiles
+
+
+def BuildImportLibs(flags, inputs_by_part, deffiles):
+  """Run the linker to generate an import library."""
+  import_libs = []
+  Log('building import libs')
+  for i, (inputs, deffile) in enumerate(zip(inputs_by_part, deffiles)):
+    libfile = 'part%d.lib' % i
+    flags_with_implib_and_deffile = flags + ['/IMPLIB:%s' % libfile,
+                                             '/DEF:%s' % deffile]
+    RunLinker(flags_with_implib_and_deffile, i, inputs, 'implib')
+    import_libs.append(libfile)
+  return import_libs
+
+
+def AttemptLink(flags, inputs_by_part, unresolved_by_part, deffiles,
+                import_libs):
+  """Try to run the linker for all parts using the current round of
+  generated import libs and .def files. If the link fails, update the
+  unresolved externals list per part."""
+  dlls = []
+  all_succeeded = True
+  new_externals = []
+  Log('unresolveds now: %r' % [len(part) for part in unresolved_by_part])
+  for i, (inputs, deffile) in enumerate(zip(inputs_by_part, deffiles)):
+    Log('running link, part %d' % i)
+    others_implibs = import_libs[:]
+    others_implibs.pop(i)
+    inputs_with_implib = inputs + filter(lambda x: x, others_implibs)
+    if deffile:
+      flags = flags + ['/DEF:%s' % deffile, '/LTCG']
+    stdout, rc, output = RunLinker(flags, i, inputs_with_implib, 'final')
+    if rc != 0:
+      all_succeeded = False
+      new_externals.append(ParseOutExternals(stdout))
+    else:
+      new_externals.append([])
+      dlls.append(output)
+  combined_externals = [sorted(set(prev) | set(new))
+                        for prev, new in zip(unresolved_by_part, new_externals)]
+  return all_succeeded, dlls, combined_externals
+
+
+def main():
+  flags, inputs = GetFlagsAndInputs(sys.argv[1:])
+  with open('../../build/split_link_partition.json') as partition:
+    description = eval(partition.read())

[M-A Ruel, 2013/05/14 00:39:09]

If you eval() it, it's not json. Please don't make the file extension confusing.

[scottmg, 2013/05/14 03:35:07]

Done.

+  inputs_by_part = []
+  description_parts = description['parts']
+  # We currently assume that if a symbols isn't in dll 0, then it's in dll 1
+  # when generating def files. Otherwise, we'd need to do more complex things
+  # to figure out where each symbol actually is to assign it to the correct
+  # .def file.
+  num_parts = len(description_parts)
+  assert num_parts == 2, "Can't handle > 2 dlls currently"
+  description_parts.reverse()
+  inputs_by_part = [[] for x in range(num_parts)]

[M-A Ruel, 2013/05/14 00:39:09]

[[] for _ in range(num_parts)]

This would have been flagged by pylint. It would be a good idea to make sure
this is pylint error free.

[scottmg, 2013/05/14 03:35:07]

Done.

+  for input in inputs:
+    i = PartFor(input, description_parts, description['all'])
+    if i == -1:
+      for part in inputs_by_part:
+        part.append(input)
+    else:
+      inputs_by_part[i].append(input)
+  inputs_by_part.reverse()
+
+  unresolved_by_part = [[] for x in range(num_parts)]
+  import_libs = [None] * num_parts
+  deffiles = [None] * num_parts
+
+  for i in range(5):
+    Log('--- starting pass %d' % i)
+    ok, dlls, unresolved_by_part = AttemptLink(
+        flags, inputs_by_part, unresolved_by_part, deffiles, import_libs)
+    if ok:
+      break
+    deffiles = GenerateDefFiles(unresolved_by_part)
+    import_libs = BuildImportLibs(flags, inputs_by_part, deffiles)
+  else:
+    raise SystemExit('Failed to link.')

[M-A Ruel, 2013/05/14 00:39:09]

?
return 1

[scottmg, 2013/05/14 03:35:07]

Done.

+  Log('built %r' % dlls)
+
+  return 0
+
+
+if __name__ == '__main__':
+  sys.exit(main())