Subzero: Run cross tests as a much more configurable python script.

pydir/crosstest_generator.py

Index: pydir/crosstest_generator.py
diff --git a/pydir/crosstest_generator.py b/pydir/crosstest_generator.py
new file mode 100755
index 0000000000000000000000000000000000000000..7264818f1d75865691a67bdab49fcaba3e0faeb6
--- /dev/null
+++ b/pydir/crosstest_generator.py
@@ -0,0 +1,210 @@
+#!/usr/bin/env python2
+
+import argparse
+import os
+import shutil
+import sys
+
+from utils import shellcmd
+from utils import FindBaseNaCl
+
+def Match(desc, includes, excludes, default_match):
+  """Determines whether desc is a match against includes and excludes.
+
+  'desc' is a set of attributes, and 'includes' and 'excludes' are lists of sets
+  of attributes.
+
+  If 'desc' matches any element from 'excludes', the result is False.
+  Otherwise, if 'desc' matches any element from 'includes', the result is True.
+  Otherwise, the 'default_match' value is returned.
+  """
+  for exclude in excludes:
+    if exclude < desc:
+      return False
+  for include in includes:
+    if include < desc:
+      return True
+  return default_match
+
+def main():
+  """Framework for cross test generation and execution.
+
+  Builds and executes cross tests from the space of all possible attribute
+  combinations.  The space can be restricted by providing subsets of attributes
+  to specifically include or exclude.
+  """
+  # Defines the set of tests and each test's inputs.  Each test maps to a
+  # dictionary containing a single driver file, one or more test source files,
+  # and an optional list of crosstest.py flags for special cases.
+  tests = {

[Mircea Trofin, 2015/03/08 16:23:30]

will this list potentially grow? If so, is the expectation the developer edit
this file? In that case, could we simplify this by, for example, requiring a
naming pattern (like there appears to be one, too), and looking in a certain
directory for the tests?

[Jim Stichnoth, 2015/03/09 18:16:16]

Good point that I should try not to hard-code the tests into the driver script.

I'd prefer not to impose file naming conventions on the tests, and there's also
the issue where test_sync_atomic needs additional flags.  Instead, I essentially
moved this dictionary into a separate config file.

+    'simple_loop' : {
+      'driver' : 'simple_loop_main.c',
+      'test' : [ 'simple_loop.c' ]
+    },
+    'mem_intrin' : {
+      'driver' : 'mem_intrin_main.cpp',
+      'test' : [ 'mem_intrin.cpp' ]
+    },
+    'test_arith' : {
+      'driver' : 'test_arith_main.cpp',
+      'test' : [ 'test_arith.cpp', 'test_arith_frem.ll', 'test_arith_sqrt.ll' ]
+    },
+    'test_bitmanip' : {
+      'driver' : 'test_bitmanip_main.cpp',
+      'test' : [ 'test_bitmanip.cpp', 'test_bitmanip_intrin.ll' ]
+    },
+    'test_calling_conv' : {
+      'driver' : 'test_calling_conv_main.cpp',
+      'test' : [ 'test_calling_conv.cpp' ]
+    },
+    'test_cast' : {
+      'driver' : 'test_cast_main.cpp',
+      'test' : [ 'test_cast.cpp', 'test_cast_to_u1.ll', 'test_cast_vectors.ll' ]
+    },
+    'test_fcmp' : {
+      'driver' : 'test_fcmp_main.cpp',
+      'test' : [ 'test_fcmp.pnacl.ll' ]
+    },
+    'test_global' : {
+      'driver' : 'test_global_main.cpp',
+      'test' : [ 'test_global.cpp' ]
+    },
+    'test_icmp' : {
+      'driver' : 'test_icmp_main.cpp',
+      'test' : [ 'test_icmp.cpp', 'test_icmp_i1vec.ll' ]
+    },
+    'test_select' : {
+      'driver' : 'test_select_main.cpp',
+      'test' : [ 'test_select.ll' ]
+    },
+    'test_stacksave' : {
+      'driver' : 'test_stacksave_main.c',
+      'test' : [ 'test_stacksave.c' ]
+    },
+    'test_sync_atomic' : {
+      'driver' : 'test_sync_atomic_main.cpp',
+      # Compile the non-subzero object files straight from source since the
+      # native LLVM backend does not understand how to lower NaCl-specific
+      # intrinsics.
+      'flags' : [ '--crosstest-bitcode=0' ],
+      'test' : [ 'test_sync_atomic.cpp' ]
+    },
+    'test_vector_ops' : {
+      'driver' : 'test_vector_ops_main.cpp',
+      'test' : [ 'test_vector_ops.ll' ]
+    },
+  }
+  # The rest of the attribute sets.
+  targets = [ 'x8632' ]
+  sandboxing = [ 'native', 'sandbox' ]
+  opt_levels = [ 'Om1', 'O2' ]
+  arch_attrs = [ 'sse2', 'sse4.1' ]
+  # all_keys is only used in the help text.
+  all_keys = '; '.join([' '.join(sorted(tests.keys())), ' '.join(targets),
+                        ' '.join(sandboxing), ' '.join(opt_levels),
+                        ' '.join(arch_attrs)])
+
+  root = FindBaseNaCl()
+  argparser = argparse.ArgumentParser(
+    description='  ' + main.__doc__ +
+    'The set of attributes is the following:\n\n' + all_keys,
+    formatter_class=argparse.RawTextHelpFormatter)
+  argparser.add_argument('--include', '-i', default=[], dest='include',
+                         action='append', metavar='ATTR_LIST',
+                         help='Attributes to include (comma-separated). ' +
+                              'Can be used multiple times.')
+  argparser.add_argument('--exclude', '-e', default=[], dest='exclude',
+                         action='append', metavar='ATTR_LIST',
+                         help='Attributes to include (comma-separated). ' +
+                              'Can be used multiple times.')
+  argparser.add_argument('--verbose', '-v', default=False, action='store_true',
+                         help='Use verbose output')
+  argparser.add_argument('--defer', default=False, action='store_true',
+                         help='Defer execution until all executables are built')
+  argparser.add_argument('--no-compile', '-n', default=False,
+                         action='store_true',
+                         help="Don't build; reuse binaries from the last run")
+  argparser.add_argument('--dir', dest='dir', metavar='DIRECTORY',
+                         default=('{root}/toolchain_build/src/subzero/' +
+                                  'crosstest/Output').format(root=root),
+                         help='Output directory')
+  argparser.add_argument('--lit', default=False, action='store_true',
+                         help='Generate files for lit testing')
+  args = argparser.parse_args()
+  # includes and excludes are both lists of sets.
+  includes = [ set(item.split(',')) for item in args.include ]
+  excludes = [ set(item.split(',')) for item in args.exclude ]
+  # If any --include args are provided, the default is to not match.
+  default_match = not args.include
+
+  # Delete and recreate the output directory, unless --no-compile was specified.
+  if not args.no_compile:
+    if os.path.exists(args.dir):
+      if os.path.isdir(args.dir):
+        shutil.rmtree(args.dir)
+      else:
+        os.remove(args.dir)
+    if not os.path.exists(args.dir):
+      os.makedirs(args.dir)
+
+  # pypath is where to find other Subzero python scripts.
+  pypath = os.path.abspath(os.path.dirname(sys.argv[0]))
+  # Run from the crosstest directory to make it easy to grab inputs.
+  crosstest_dir = '{root}/toolchain_build/src/subzero/crosstest'.format(
+    root=root)
+  os.chdir(crosstest_dir)
+  # If --defer is specified, collect the run commands into deferred_cmds for
+  # later execution.
+  deferred_cmds = []
+  for test in sorted(tests.keys()):
+    for target in targets:
+      for sb in sandboxing:
+        for opt in opt_levels:
+          for attr in arch_attrs:
+            desc = [ test, target, sb, opt, attr ]
+            if Match(set(desc), includes, excludes, default_match):
+              exe = '{test}_{target}_{sb}_{opt}_{attr}'.format(
+                test=test, target=target, sb=sb, opt=opt,
+                attr=attr)
+              extra = tests[test]['flags'] if 'flags' in tests[test] else []

[jvoung (off chromium), 2015/03/09 17:34:21]

Could probably have done "tests[test].get('flags', [])" ?

[Jim Stichnoth, 2015/03/09 18:16:16]

Hmm, nice.  Unfortunately, I don't think there's an equivalent for the new
ConfigParser version of this?

[jvoung (off chromium), 2015/03/09 20:34:59]

Hmm I don't see anything like that for ConfigParser, so this is fine =)

+              # Generate the compile command.
+              cmp_cmd = ['{path}/crosstest.py'.format(path=pypath),
+                         '-{opt}'.format(opt=opt),
+                         '--mattr={attr}'.format(attr=attr),
+                         '--prefix=Subzero_',
+                         '--target={target}'.format(target=target),
+                         '--sandbox={sb}'.format(sb='1' if sb=='sandbox'
+                                                 else '0'),
+                         '--dir={dir}'.format(dir=args.dir),
+                         '--output={exe}'.format(exe=exe),
+                         '--driver={drv}'.format(drv=tests[test]['driver']),
+                        ] + extra + \
+                        [ '--test=' + t for t in tests[test]['test'] ]
+              run_cmd_base = os.path.join(args.dir, exe)
+              # Generate the run command.
+              run_cmd = run_cmd_base
+              if sb == 'sandbox':
+                run_cmd = '{root}/run.py -q '.format(root=root) + run_cmd
+              if args.lit:
+                # Create a file to drive the lit test.
+                with open(run_cmd_base + '.xtest', 'w') as f:
+                  f.write('# RUN: sh %s | FileCheck %s\n')
+                  f.write('cd ' + crosstest_dir + ' && \\\n')
+                  f.write(' '.join(cmp_cmd) + ' && \\\n')
+                  f.write(run_cmd + '\n')
+                  f.write('echo Recreate a failure using ' + __file__ +
+                          ' --include=' + ','.join(desc) + '\n')
+                  f.write('# CHECK: Failures=0\n')
+              else:
+                if not args.no_compile:
+                  shellcmd(cmp_cmd,
+                           echo=args.verbose)
+                if (args.defer):
+                  deferred_cmds.append(run_cmd)
+                else:
+                  shellcmd(run_cmd, echo=True)
+  for run_cmd in deferred_cmds:
+    shellcmd(run_cmd, echo=True)
+
+if __name__ == '__main__':
+  main()