Add BoringSSL GYP files.

third_party/boringssl/update_gypi_and_asm.py

+# Copyright (c) 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can b
+# found in the LICENSE file.
+
+# This utility enumerates the BoringSSL source in src/ and generates two gypi

[M-A Ruel, 2014/07/09 16:56:43]

Convert to docstring, e.g.

"""Enumerates the BoringSSL ..."""

[agl, 2014/07/09 17:49:23]

Done.

+# files: boringssl.gypi and boringssl_tests.gypi
+
+import os
+import stat
+import subprocess
+import sys
+
+
+# OS_ARCH_COMBOS maps from OS and platform to the OpenSSL assembly "style" for
+# that platform and the extension used by asm files.
+OS_ARCH_COMBOS = [
+    ('linux', 'arm', 'elf', 'S'),

[M-A Ruel, 2014/07/09 16:56:43]

You are free to align at +2 if you prefer.

[agl, 2014/07/09 17:49:23]

Am just running pyformat over it and letting it decide.

+    ('linux', 'x86', 'elf', 'S'),
+    ('linux', 'x86_64', 'elf', 'S'),
+    ('mac', 'x86', 'macosx', 'S'),
+    ('mac', 'x86_64', 'macosx', 'S'),
+    ('win', 'x86_64', 'masm', 'asm'),
+]
+
+# NON_PERL_FILES enumerates assembly files that are not processed by the
+# perlasm system.
+NON_PERL_FILES = {
+    ('linux', 'arm'): [
+        'src/crypto/poly1305/poly1305_arm_asm.S',
+        'src/crypto/chacha/chacha_vec_arm.S',
+    ],
+}
+
+
+def FindCMakeFiles(directory):
+  """Returns list of all CMakeLists.txt files recursively in directory."""
+  cmakefiles = []
+
+  for (path, _, filenames) in os.walk(directory):
+    for filename in filenames:
+      if filename == 'CMakeLists.txt':
+        cmakefiles.append(os.path.join(path, filename))
+
+  return cmakefiles
+
+
+def NoTests(dent, is_dir):
+  '''NoTests is a filter function that can be passed to FindCFiles in order to

[M-A Ruel, 2014/07/09 16:56:44]

""" for docstrings, everywhere for consistency.

[agl, 2014/07/09 17:49:24]

Done.

+  remove test sources.'''
+  if is_dir:
+    return dent != 'test'
+  return 'test.' not in dent and not dent.startswith('example_')
+
+
+def OnlyTests(dent, is_dir):
+  '''OnlyTests is a filter function that can be passed to FindCFiles in order to
+  remove non-test sources.'''
+  if is_dir:
+    return True
+  return '_test.' in dent or dent.startswith('example_')
+
+
+def FindCFiles(directory, filter_func):
+  '''Recurses through directory and returns a list of paths to all the C source
+  files that pass filter_func.'''
+  cfiles = []
+
+  for (path, dirnames, filenames) in os.walk(directory):
+    for filename in filenames:
+      if filename.endswith('.c') and filter_func(filename, False):
+        cfiles.append(os.path.join(path, filename))
+        continue
+
+    for (i, dirname) in enumerate(dirnames):
+      if not filter_func(dirname, True):
+        del dirnames[i]
+
+  return cfiles
+
+
+def ExtractPerlAsmFromCMakeFile(cmakefile):
+  '''Parses the contents of the CMakeLists.txt file passed as an argument and
+  returns a list of all the perlasm() directives found in the file.'''
+  perlasms = []
+  with open(cmakefile) as f:
+    for line in f:
+      line = line.strip()
+      if not line.startswith('perlasm('):
+        continue
+      if not line.endswith(')'):
+        raise ValueError('Bad perlasm line in %s' % cmakefile)
+      # Remove "perlasm(" from start and ")" from end
+      params = line[8:-1].split()
+      if len(params) < 2:
+        raise ValueError('Bad perlasm line in %s' % cmakefile)
+      perlasms.append({
+          'extra_args': params[2:],
+          'input': os.path.join(os.path.dirname(cmakefile), params[1]),
+          'output': os.path.join(os.path.dirname(cmakefile), params[0]),
+      })
+
+  return perlasms
+
+
+def ReadPerlAsmOperations():
+  '''Returns a list of all perlasm() directives found in CMake config files in
+  src/.'''
+  perlasms = []
+  cmakefiles = FindCMakeFiles('src')

[M-A Ruel, 2014/07/09 16:56:43]

import itertools

...

return itertools.chain.from_iterable(
    ExtractPerlAsmFromCMakeFile(i) for i in FindCMakeFiles('src'))

[agl, 2014/07/09 17:49:24]

I'm sure that this is far more efficient, but I think the existing code is
clearer and it's only dealing with lists of a couple of dozen entries.

+
+  for cmakefile in cmakefiles:
+    perlasms.extend(ExtractPerlAsmFromCMakeFile(cmakefile))
+
+  return perlasms
+
+
+def PerlAsm(output_filename, input_filename, perlasm_style, extra_args):
+  '''PerlAsm runs the a perlasm script and puts the output into
+  output_filename.'''
+  try:
+    os.makedirs(os.path.dirname(output_filename))
+  except OSError:

[M-A Ruel, 2014/07/09 16:56:43]

I prefer for you to use:
base_dir = os.path.dirname(output_filename)
if not os.path.isdir(base_dir):
  os.makedirs(base_dir)

so that if there is an access control error, it won't be swallowed.

[agl, 2014/07/09 17:49:23]

Done.

+    pass
+  output = subprocess.check_output(
+      ['perl', input_filename, perlasm_style] + extra_args)
+  with open(output_filename, 'w+') as out_file:
+    out_file.write(output)
+
+
+def WriteAsmFiles(perlasms):
+  '''WriteAsmFiles generates asm files from perlasm directives for each
+  supported OS x platform combination.'''
+  asmfiles = {}
+
+  for osarch in OS_ARCH_COMBOS:
+    (osname, arch, perlasm_style, asm_ext) = osarch
+    key = (osname, arch)
+    outDir = '%s-%s' % key
+
+    for perlasm in perlasms:
+      filename = os.path.basename(perlasm['input'])
+      output = perlasm['output']
+      if not output.startswith('src'):
+        raise ValueError('output missing src: %s' % output)
+      output = os.path.join(outDir, output[4:])
+      output = output.replace('${ASM_EXT}', asm_ext)
+
+      found = False
+      if (arch == 'x86_64' and

[M-A Ruel, 2014/07/09 16:56:44]

if arch == 'x86_64' and ('x86_64' in filename or 'avx2' in filename):

[agl, 2014/07/09 17:49:23]

Done.

+          ('x86_64' in filename or 'avx2' in filename)):
+        found = True
+      elif (arch == 'x86' and 'x86' in filename and 'x86_64' not in filename):

[M-A Ruel, 2014/07/09 16:56:44]

elif arch == 'x86' and 'x86' in filename and 'x86_64' not in filename:

[agl, 2014/07/09 17:49:23]

Done.

+        found = True
+      elif (arch == 'arm' and 'arm' in filename):

[M-A Ruel, 2014/07/09 16:56:44]

elif arch == 'arm' and 'arm' in filename:

[agl, 2014/07/09 17:49:23]

Done.

+        found = True
+
+      if found:
+        PerlAsm(output, perlasm['input'], perlasm_style, perlasm['extra_args'])
+        asmfiles[key] = asmfiles.get(key, []) + [output]

[M-A Ruel, 2014/07/09 16:56:44]

I prefer:
asmfiles.setdefault(key, []).append(output)

it's a tad shorter and easier to glance at for me.

[agl, 2014/07/09 17:49:23]

Done.

+
+  for (key, non_perl_asm_files) in NON_PERL_FILES.iteritems():
+    asmfiles[key] = asmfiles.get(key, []) + non_perl_asm_files

[M-A Ruel, 2014/07/09 16:56:43]

asmfiles.setdefault(key, []).extend(non_perl_asm_files)

[agl, 2014/07/09 17:49:24]

Done.

+
+  return asmfiles
+
+
+def PrintVariableSection(out, name, files):
+  out.write("""    '%s': [\n""" % name)
+  files.sort()
+  for f in files:

[M-A Ruel, 2014/07/09 16:56:44]

for f in sorted(files):

would prevent from modifying input parameters.

[agl, 2014/07/09 17:49:24]

Done.

+    out.write("""      '%s',\n""" % f)
+  out.write('''    ],\n''')
+
+
+def main():
+  crypto_c_files = FindCFiles(os.path.join('src', 'crypto'), NoTests)
+  ssl_c_files = FindCFiles(os.path.join('src', 'ssl'), NoTests)
+
+  gypi = file('boringssl.gypi', 'w+')

[M-A Ruel, 2014/07/09 16:56:44]

with open('boringssl.gypi', 'w+') as gypi:

[agl, 2014/07/09 17:49:23]

Done.

+
+  header = '''# Copyright (c) 2014 The Chromium Authors. All rights reserved.

[M-A Ruel, 2014/07/09 16:56:44]

Could you move this out of the function? I find multiline strings significantly
break the reading flow (and confuse the hell out of my editor).

[agl, 2014/07/09 17:49:23]

Done.

+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# This file is created by update_gypi_and_asm.py. Do not edit manually.
+
+'''
+
+  gypi.write(header + '''{
+  'variables': {\n''')
+
+  PrintVariableSection(
+      gypi, 'boringssl_lib_sources', crypto_c_files + ssl_c_files)
+
+  perlasms = ReadPerlAsmOperations()
+  asmfiles = WriteAsmFiles(perlasms)
+
+  for ((osname, arch), asm_files) in asmfiles.items():

[M-A Ruel, 2014/07/09 16:56:44]

for ((osname, arch), asm_files) in sorted(WriteAsmFiles(perlasms).iteritems()):

I recommend to sort to that the written file is deterministic. As a matter of
fact, I highly recommend to sort everything even if not strictly needed, so that
the output is deterministic. Otherwise I'll have to fix it up later myself due
to
http://www.chromium.org/developers/testing/isolated-testing/deterministic-builds
:)

[agl, 2014/07/09 17:49:23]

Done. Although note that this script isn't running at build time, it'll only be
rerun by hand when the set of sources needs to be updated.

+    PrintVariableSection(gypi, 'boringssl_%s_%s_sources' %
+                         (osname, arch), asm_files)
+
+  gypi.write('''  }\n}\n''')
+
+  gypi.close()
+
+  test_c_files = FindCFiles(os.path.join('src', 'crypto'), OnlyTests)
+
+  test_gypi = file('boringssl_tests.gypi', 'w+')

[M-A Ruel, 2014/07/09 16:56:44]

same

[agl, 2014/07/09 17:49:23]

Done.

+
+  test_gypi.write(header + '''{
+  'targets': [\n''')
+
+  test_names = []
+  for test in test_c_files:
+    test_name = 'boringssl_%s' % os.path.splitext(os.path.basename(test))[0]
+    test_gypi.write('''    {
+      'target_name': '%s',
+      'type': 'executable',
+      'dependencies': [
+        'boringssl',
+      ],
+      'sources': [
+        '%s',
+      ],
+    },\n''' % (test_name, test))
+    test_names.append(test_name)
+
+  test_names.sort()
+
+  test_gypi.write('''  ],
+  'variables': {
+    'boringssl_test_targets': [\n''')
+
+  for test in test_names:
+    test_gypi.write("""      '%s',\n""" % test)
+
+  test_gypi.write('''    ],
+  }\n}\n''')
+
+  test_gypi.close()
+
+  return 0
+
+if __name__ == '__main__':
+  sys.exit(main())