split_link tool, config, and scripts for windows build

tools/win/split_link/split_link.py

+# Copyright 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Takes the same arguments as Windows link.exe, and a definition of libraries
+to split into subcomponents. Does multiple passes of link.exe invocation to
+determine exports between parts and generates .def and import libraries to
+cause symbols to be available to other parts."""
+
+from ctypes import *
+import os
+import re
+import subprocess
+import sys
+
+
+def Log(message):
+  print 'split_link:', message
+
+
+def GetFlagsAndInputs(argv):
+  """Parse the command line intended for link.exe and return the flags and

[M-A Ruel, 2013/05/13 21:55:59]

In general, docstrings should start with an imperative verb.

[scottmg, 2013/05/13 23:00:15]

Sorry, don't understand this one. You want "Parses"?
https://www.google.com/#q=parse&tbs=dfn:1&tbo=u&sa=X&ei=B2ORUayRNrT8iQKh2YCwB...

[M-A Ruel, 2013/05/14 00:39:09]

Yes.
http://google-styleguide.googlecode.com/svn/trunk/pyguide.html?showone=Commen...

+  input files."""
+  rsp_expanded = []
+  for arg in argv:
+    if arg[0] == '@':
+      with open(arg[1:]) as rsp:
+        rsp_expanded.append(rsp.read().replace('\r', '').replace('\n', ' '))

[M-A Ruel, 2013/05/13 21:55:59]

open() should remove any CR when reading in text mode, which is the default, so
the first replace should be unneeded.

So in the end you could do:
rsp_expanded.extend(rsp.read().splitlines())

I think it'd be slightly faster overall

[scottmg, 2013/05/13 23:00:15]

Done.

+    else:
+      rsp_expanded.append(arg)
+
+  # Use CommandLineToArgvW so we match link.exe parsing (I think?).
+  size = c_int()
+  ptr = windll.shell32.CommandLineToArgvW(
+      create_unicode_buffer(' '.join(rsp_expanded)),
+      byref(size))
+  ref = c_wchar_p * size.value
+  raw = ref.from_address(ptr)
+  args = [arg for arg in raw]

[M-A Ruel, 2013/05/13 21:55:59]

It's not clear why you need args as a named variable before the call.
Personally, I'd do a tight try/finally to make this clearer.

[scottmg, 2013/05/13 23:00:15]

Done.

+  windll.kernel32.LocalFree(ptr)
+
+  inputs = []
+  flags = []
+  for arg in args:
+    lower_arg = arg.lower()
+    # We'll be replacing this ourselves.
+    if lower_arg.startswith('/out:'):
+      continue
+    if (not lower_arg.startswith('/') and
+        (lower_arg.endswith('.obj') or

[M-A Ruel, 2013/05/13 21:55:59]

lower_arg.endswith(('.lib', '.obj', '.res'))

[scottmg, 2013/05/13 23:00:15]

http://docs.python.org/2/library/stdtypes.html#str.endswith "Changed in version
2.5: Accept tuples as suffix." Magic!

Done.

+         lower_arg.endswith('.lib') or
+         lower_arg.endswith('.res'))):
+      inputs.append(arg)
+    else:
+      flags.append(arg)
+
+  return flags, inputs
+
+
+def GetOriginalLinkerPath():
+  _winreg = None
+  if sys.platform == 'win32':
+    import _winreg
+  elif sys.platform == 'cygwin':
+    import cygwinreg as _winreg
+
+  try:
+    val = _winreg.QueryValue(_winreg.HKEY_CURRENT_USER,
+                             'Software\\Chromium\\split_link_installed')
+    if os.path.exists(val):
+      return val
+  except WindowsError:
+    pass
+
+  raise SystemExit("Couldn't read linker location from registry")
+
+
+def PartFor(input, description_parts, description_all):
+  """Determine which part a given link input should be put into (or all)."""
+  # Check if it should go in all parts.
+  input = input.lower()

[M-A Ruel, 2013/05/13 21:55:59]

I don't like using input as a variable name since it is a builtin.
http://docs.python.org/2/library/functions.html#input

[scottmg, 2013/05/13 23:00:15]

file? ;) Done.

[M-A Ruel, 2013/05/14 00:39:09]

Yeah, I recommend avoiding "file" too.

+  for spec in description_all:

[M-A Ruel, 2013/05/13 21:55:59]

if any(re.search(spec, input) for spec in description_all):
  return -1

[scottmg, 2013/05/13 23:00:15]

Done.

+    if re.search(spec, input):
+      return -1
+  # Or pick which particular one it belongs in.
+  for i, spec_list in enumerate(description_parts):
+    for spec in spec_list:

[M-A Ruel, 2013/05/13 21:55:59]

if any(re.search(spec, input) for spec in spec_list):
  return i

[scottmg, 2013/05/13 23:00:15]

Done.

+      if re.search(spec, input):
+        return i
+  raise ValueError("couldn't find location for %s" % input)
+
+
+def ParseOutExternals(output):
+  """Given the stdout of link.exe, parse the errors messages to retrieve all
+  symbols that are unresolved."""
+  result = set()
+  # Styles of messages for unresolved externals, and a boolean to indicate
+  # whether the error message emits the symbols with or without a leading
+  # underscore.
+  unresolved_regexes = [
+    (re.compile(r' : error LNK2019: unresolved external symbol ".*" \((.*)\)'
+                r' referenced in function'),
+     False),
+    (re.compile(r' : error LNK2001: unresolved external symbol ".*" \((.*)\)$'),
+     False),
+    (re.compile(r' : error LNK2019: unresolved external symbol (.*)'
+                r' referenced in function '),
+     True),
+    (re.compile(r' : error LNK2001: unresolved external symbol (.*)$'),
+     True),
+  ]
+  for line in output.splitlines():
+    line = line.strip()
+    for i, (regex, strip_leading_underscore) in enumerate(unresolved_regexes):
+      mo = regex.search(line)
+      if mo:
+        if strip_leading_underscore:
+          result.add(mo.group(1)[1:])
+        else:
+          result.add(mo.group(1))
+        break
+
+  mo = re.search(r'fatal error LNK1120: (\d+) unresolved externals', output)
+  # Make sure we have the same number that the linker thinks we have.
+  assert mo or not result
+  if len(result) != int(mo.group(1)):
+    print output
+    print 'Expecting %d, got %d' % (int(mo.group(1)), len(result))
+  assert len(result) == int(mo.group(1))
+  return sorted(result)
+
+
+def AsCommandLineArgs(list):
+  """Intended for output to a response file. Quotes all arguments."""

[M-A Ruel, 2013/05/13 21:55:59]

What about an arg that is already quoted? If you don't think it'll ever happen,
it's fine to leave it as-is

[scottmg, 2013/05/13 23:00:15]

They shouldn't be quoted because they've come from the output of
CommandLineToArgvW.

+  return '\n'.join('"' + x + '"' for x in list)
+
+
+def RunLinker(flags, index, inputs, phase):
+  """Invoke the linker and return the stdout, returncode and target name."""
+  rspfile = 'part%d_%s.rsp' % (index, phase)
+  import os
+  with open(rspfile, 'w') as f:

[M-A Ruel, 2013/05/13 21:55:59]

You may want to force CRLF EOL if you plan to have this script running on
cygwin. I don't know if link.exe accepts LF EOL files.

[scottmg, 2013/05/13 23:00:15]

Not sure if it'll work on cygwin (ctypes?). This is mostly Official-only so I'm
OK with limiting to depot_tools python.

[M-A Ruel, 2013/05/14 00:39:09]

Then you should make it look like you are trying to support cygwin, like when
you import cygwinreg.

+    print >>f, AsCommandLineArgs(inputs)
+    print >>f, AsCommandLineArgs(flags)
+    output_name = 'chrome%d.dll' % index
+    print >>f, '/ENTRY:ChromeEmptyEntry@12'
+    print >>f, '/OUT:' + output_name
+  # Log('[[[\n' + open(rspfile).read() + '\n]]]')
+  link_exe = GetOriginalLinkerPath()
+  popen = subprocess.Popen(
+      [link_exe, '@' + rspfile], stdout=subprocess.PIPE, shell=True)

[M-A Ruel, 2013/05/13 21:55:59]

shell=True is really needed?

[scottmg, 2013/05/13 23:00:15]

Done.

+  stdout, _ = popen.communicate()
+  return stdout, popen.returncode, output_name
+
+
+def GenerateDefFiles(unresolved_by_part):
+  """Given a list of unresolved externals, generate a .def file that will
+  cause all those symbols to be exported."""
+  deffiles = []
+  Log('generating .def files')
+  for i, part in enumerate(unresolved_by_part):
+    deffile = 'part%d.def' % i
+    with open(deffile, 'w') as f:
+      print >>f, 'LIBRARY chrome%d.dll' % i
+      print >>f, 'EXPORTS'
+      for j, part in enumerate(unresolved_by_part):
+        if i == j:
+          continue
+        print >>f, '\n'.join('  ' + export for export in part)
+    deffiles.append(deffile)
+  return deffiles
+
+
+def BuildImportLibs(flags, inputs_by_part, deffiles):
+  """Run the linker to generate an import library."""
+  import_libs = []
+  Log('building import libs')
+  for i, (inputs, deffile) in enumerate(zip(inputs_by_part, deffiles)):
+    libfile = 'part%d.lib' % i
+    flags_with_implib_and_deffile = flags + ['/IMPLIB:%s' % libfile,
+                                             '/DEF:%s' % deffile]
+    RunLinker(flags_with_implib_and_deffile, i, inputs, 'implib')
+    import_libs.append(libfile)
+  return import_libs
+
+
+def AttemptLink(flags, inputs_by_part, unresolved_by_part, deffiles,
+                import_libs):
+  """Try to run the linker for all parts using the current round of
+  generated import libs and .def files. If the link fails, update the
+  unresolved externals list per part."""
+  dlls = []
+  all_succeeded = True
+  new_externals = []
+  Log('unresolveds now: %r' % [len(part) for part in unresolved_by_part])
+  for i, (inputs, deffile) in enumerate(zip(inputs_by_part, deffiles)):
+    Log('running link, part %d' % i)
+    others_implibs = import_libs[:]
+    others_implibs.pop(i)
+    inputs_with_implib = inputs + filter(lambda x: x, others_implibs)
+    if deffile:
+      flags = flags + ['/DEF:%s' % deffile, '/LTCG']
+    stdout, rc, output = RunLinker(flags, i, inputs_with_implib, 'final')
+    if rc != 0:
+      all_succeeded = False
+      new_externals.append(ParseOutExternals(stdout))
+    else:
+      new_externals.append([])
+      dlls.append(output)
+  combined_externals = [sorted(set(prev) | set(new))
+                        for prev, new in zip(unresolved_by_part, new_externals)]
+  return all_succeeded, dlls, combined_externals
+
+
+def main():
+  flags, inputs = GetFlagsAndInputs(sys.argv[1:])
+  with open('../../build/split_link_partition.json') as partition:
+    description = eval(partition.read())
+  inputs_by_part = []
+  description_parts = description['parts']
+  # We currently assume that if a symbols isn't in dll 0, then it's in dll 1
+  # when generating def files. Otherwise, we'd need to do more complex things
+  # to figure out where each symbol actually is to assign it to the correct
+  # .def file.
+  num_parts = len(description_parts)
+  assert num_parts == 2, "Can't handle > 2 dlls currently"
+  description_parts.reverse()
+  inputs_by_part = [[] for x in range(num_parts)]
+  for input in inputs:
+    i = PartFor(input, description_parts, description['all'])
+    if i == -1:
+      for part in inputs_by_part:
+        part.append(input)
+    else:
+      inputs_by_part[i].append(input)
+  inputs_by_part.reverse()
+
+  unresolved_by_part = [[] for x in range(num_parts)]
+  import_libs = [None] * num_parts
+  deffiles = [None] * num_parts
+
+  for i in range(5):
+    Log('--- starting pass %d' % i)
+    ok, dlls, unresolved_by_part = AttemptLink(

[M-A Ruel, 2013/05/13 21:55:59]

Could it be possible to make it deterministic?

[scottmg, 2013/05/13 23:00:15]

Not that I could figure out. The linker seems to traverse something like
breadth-first and so stops emitting failures after a certain point. As mention
in the CL description, it doesn't seem easy to get the errors in a different way
either.

+        flags, inputs_by_part, unresolved_by_part, deffiles, import_libs)
+    if ok:
+      break
+    deffiles = GenerateDefFiles(unresolved_by_part)
+    import_libs = BuildImportLibs(flags, inputs_by_part, deffiles)
+  else:
+    raise SystemExit('Failed to link.')
+  Log('built %r' % dlls)
+

[M-A Ruel, 2013/05/13 21:55:59]

return 0

[scottmg, 2013/05/13 23:00:15]

Done.

+
+if __name__ == '__main__':
+  sys.exit(main())