Subzero - WASM: Codegen fixes, better test infrastructure

pydir/wasm-run-torture-tests.py

 #!/usr/bin/env python2
 
 #===- subzero/wasm-run-torture-tests.py - Subzero WASM Torture Test Driver ===//
 #
 #                        The Subzero Code Generator
 #
 # This file is distributed under the University of Illinois Open Source
 # License. See LICENSE.TXT for details.
 #
 #===-----------------------------------------------------------------------===//
 
+from __future__ import print_function
+import argparse
 import glob
+import multiprocessing
 import os
+import Queue
 import shutil
+import StringIO
 import sys
+import threading
 
-IGNORED_TESTS = [
+IGNORED_TESTS = set([
   'loop-2f.c.wasm',       # mmap not in MVP
   'loop-2g.c.wasm',       # mmap not in MVP
 
-  '960521-1.c.wasm',      # sbrk not in MVP
+  '960521-1.c.wasm',      # sbrk
+  'ipa-sra-2.c.wasm',     # sbrk
+  'pr41463.c.wasm',       # sbrk
+  '20051113-1.c.wasm',    # sbrk
+  '990628-1.c.wasm',      # sbrk
+  'pr41395-2.c.wasm',     # sbrk
+  'pr42614.c.wasm',       # sbrk
+  'pr41395-1.c.wasm',     # sbrk
+  '920810-1.c.wasm',      # sbrk
+  '20000914-1.c.wasm',    # sbrk
+  'pr15262-1.c.wasm',     # sbrk
+  '941014-2.c.wasm',      # sbrk
+  'va-arg-21.c.wasm',     # sbrk
+  '20020406-1.c.wasm',    # sbrk
 
-  'pr36765.c.wasm',       # __builtin_malloc not allowed
+  # waterfall known failures
+  '20010122-1.c.wasm',
+  '20031003-1.c.wasm',
+  '20071018-1.c.wasm',
+  '20071120-1.c.wasm',
+  '20071220-1.c.wasm',
+  '20071220-2.c.wasm',
+  '20101011-1.c.wasm',
+  'alloca-1.c.wasm',
+  'bitfld-3.c.wasm',
+  'bitfld-5.c.wasm',
+  'builtin-bitops-1.c.wasm',
+  'conversion.c.wasm',
+  'eeprof-1.c.wasm',
+  'frame-address.c.wasm',
+  'pr17377.c.wasm',
+  'pr32244-1.c.wasm',
+  'pr34971.c.wasm',
+  'pr36765.c.wasm',
+  'pr39228.c.wasm',
+  'pr43008.c.wasm',
+  'pr47237.c.wasm',
+  'pr60960.c.wasm',
+  'va-arg-pack-1.c.wasm',
 
-  'pr24716.c.wasm',       # infinite loop
-  'vla-dealloc-1.c.wasm', # infinite loop
-  '20040811-1.c.wasm',    # infinite loop
-  '961125-1.c.wasm',      # infinite loop
-  '980506-1.c.wasm',      # infinite loop
-  '20070824-1.c.wasm',    # infinite loop
-  '20140212-1.c.wasm',    # infinite loop
-  'pr59014.c.wasm',       # infinite loop
-  'pr58277-2.c.wasm',     # infinite loop
-  'pr43560.c.wasm',       # infinite loop
-  '20000818-1.c.wasm',    # infinite loop
-  '20010409-1.c.wasm',    # infinite loop
-  'loop-7.c.wasm',        # infinite loop
-  'pr34415.c.wasm',       # infinite loop
-  '20011126-2.c.wasm',    # infinite loop
-  'postmod-1.c.wasm',     # infinite loop
-  'pr17133.c.wasm',       # infinite loop
-  '20021024-1.c.wasm',    # infinite loop
-  'pr15296.c.wasm',       # infinite loop
-  '990524-1.c.wasm',      # infinite loop
-  'loop-12.c.wasm',       # infinite loop
-  '961125-1.c.wasm',      # infinite loop
-]
+  '20000717-5.c.wasm', # abort() (also works without emcc)
+  '20001203-2.c.wasm', # assert fail (works without emcc)
+  '20040811-1.c.wasm', # OOB trap
+  '20070824-1.c.wasm', # abort() (also works without emcc)
+  'arith-rand-ll.c.wasm', # abort() (works without emcc)
+  'arith-rand.c.wasm', # abort() (works without emcc)
+  'pr23135.c.wasm', # OOB trap (works without emcc)
+  'pr34415.c.wasm', # (empty output?)
+  'pr36339.c.wasm', # abort() (works without emcc)
+  'pr38048-2.c.wasm', # abort() (works without emcc)
+  'pr42691.c.wasm', # abort() (works without emcc)
+  'pr43220.c.wasm', # OOB trap (works without emcc)
+  'pr43269.c.wasm', # abort() (works without emcc)
+  'vla-dealloc-1.c.wasm', # OOB trap (works without emcc)
+  '20051012-1.c.wasm', # error reading binary
+  '921208-2.c.wasm', # error reading binary
+  '920501-1.c.wasm', # error reading binary
+  'call-trap-1.c.wasm', # error reading binary
+  'pr44942.c.wasm', # error reading binary
+
+  '920625-1.c.wasm', # abort() (also fails without emcc)
+  '931004-10.c.wasm', # abort() (also fails without emcc)
+  '931004-12.c.wasm', # abort() (also fails without emcc)
+  '931004-14.c.wasm', # abort() (also fails without emcc)
+  '931004-6.c.wasm', # abort() (also fails without emcc)
+  'pr38051.c.wasm', # (empty output?) (fails without emcc)
+  'pr38151.c.wasm', # abort() (fails without emcc)
+  'pr44575.c.wasm', # abort() (fails without emcc)
+  'strct-stdarg-1.c.wasm', # abort() (fails without emcc)
+  'strct-varg-1.c.wasm', # abort() (fails without emcc)
+  'va-arg-22.c.wasm', # abort() (fails without emcc)
+  'stdarg-3.c.wasm', # abort() (fails without emcc)
+  'pr56982.c.wasm', # missing setjmp (wasm.js check did not catch)
+
+  '20010605-2.c.wasm', # missing __netf2
+  '20020413-1.c.wasm', # missing __lttf2
+  '20030914-1.c.wasm', # missing __floatsitf
+  '20040709-1.c.wasm', # missing __netf2
+  '20040709-2.c.wasm', # missing __netf2
+  '20050121-1.c.wasm', # missing __floatsitf
+  '20080502-1.c.wasm', # missing __eqtf2
+  '920501-8.c.wasm', # missing __extenddftf2
+  '930513-1.c.wasm', # missing __extenddftf2
+  '930622-2.c.wasm', # missing __floatditf
+  '960215-1.c.wasm', # missing __addtf3
+  '960405-1.c.wasm', # missing __eqtf2
+  '960513-1.c.wasm', # missing __subtf3
+  'align-2.c.wasm', # missing __eqtf2
+  'complex-6.c.wasm', # missing __subtf3
+  'complex-7.c.wasm', # missing __netf2
+  'pr49218.c.wasm', # missing __fixsfti
+  'pr54471.c.wasm', # missing __multi3
+  'regstack-1.c.wasm', # missing __addtf3
+  'stdarg-1.c.wasm', # missing __netf2
+  'stdarg-2.c.wasm', # missing __floatsitf
+  'va-arg-5.c.wasm', # missing __eqtf2
+  'va-arg-6.c.wasm', # missing __eqtf2
+  'struct-ret-1.c.wasm', # missing __extenddftf2
+])
+
+parser = argparse.ArgumentParser()
+parser.add_argument('-v', '--verbose', action='store_true')
+parser.add_argument('--translate-only', action='store_true')
+parser.add_argument('tests', nargs='*')
+args = parser.parse_args()
 
 OUT_DIR = "./build/wasm-torture"
 
+results_lock = threading.Lock()
+
 compile_count = 0
 compile_failures = []
 
 run_count = 0
 run_failures = []
 
 def run_test(test_file, verbose=False):
+  global args
   global compile_count
   global compile_failures
+  global results_lock
   global run_count
   global run_failures
   global OUT_DIR
   global IGNORED_TESTS
 
+  run_test = not args.translate_only
+
   test_name = os.path.basename(test_file)
   obj_file = os.path.join(OUT_DIR, test_name + ".o")
   exe_file = os.path.join(OUT_DIR, test_name + ".exe")
 
   if not verbose and test_name in IGNORED_TESTS:
     print("\033[1;34mSkipping {}\033[1;m".format(test_file))
     return
 
   cmd = """LD_LIBRARY_PATH=../../../../v8/out/native/lib.target ./pnacl-sz \
                -filetype=obj -target=x8632 {} -threads=0 -O2 \
                -verbose=wasm -o {}""".format(test_file, obj_file)
 
   if not verbose:
     cmd += " &> /dev/null"
 
-  sys.stdout.write(test_file + " ...");
+  out = StringIO.StringIO()
+
+  out.write(test_file + " ...");
   status = os.system(cmd);
   if status != 0:
-    print('\033[1;31m[compile fail]\033[1;m')
-    compile_failures.append(test_file)
+    print('\033[1;31m[compile fail]\033[1;m', file=out)

[Karl, 2016/04/21 19:14:17]

Are these parenthesis necessary? (More instances below).

[Eric Holk, 2016/04/21 21:56:02]

They are because I imported print_function from __future__. I did that because I
prefer the new print function's way of printing to a file other than stdout.

+    with results_lock:
+      compile_failures.append(test_file)
   else:
     compile_count += 1
 
     # Try to link and run the program.
-    cmd = "clang -g -m32 {} -o {} ./runtime/wasm-runtime.c".format(obj_file,
-                                                                   exe_file)
-    if os.system(cmd) == 0:
-      if os.system(exe_file) == 0:
-        run_count += 1
-        print('\033[1;32m[ok]\033[1;m')
+    cmd = "clang -g -m32 {} -o {} " + \
+          "./runtime/szrt.c ./runtime/wasm-runtime.cpp -lm"
+    cmd = cmd.format(obj_file, exe_file)
+
+    if not run_test or os.system(cmd) == 0:
+      if not run_test or os.system(exe_file) == 0:
+        with results_lock:
+          run_count += 1
+        print('\033[1;32m[ok]\033[1;m', file=out)
       else:
+        with results_lock:
+          run_failures.append(test_file)
+        print('\033[1;33m[run fail]\033[1;m', file=out)
+    else:
+      with results_lock:
         run_failures.append(test_file)
-        print('\033[1;33m[run fail]\033[1;m')
-    else:
-      run_failures.append(test_file)
-      print('\033[1;33m[run fail]\033[1;m')
+      print('\033[1;33m[run fail]\033[1;m', file=out)
 
-verbose = False
+  sys.stdout.write(out.getvalue())
 
-if len(sys.argv) > 1:
-  test_files = sys.argv[1:]
-  verbose = True
+verbose = args.verbose
+
+if len(args.tests) > 0:
+  test_files = args.tests
 else:
   test_files = glob.glob("./emwasm-torture-out/*.wasm")
 
 if os.path.exists(OUT_DIR):
   shutil.rmtree(OUT_DIR)
 os.mkdir(OUT_DIR)
 
+tasks = Queue.Queue()
+
+def worker():
+  while True:
+    run_test(tasks.get(), verbose)
+    tasks.task_done()
+
+for i in range(multiprocessing.cpu_count()):
+  t = threading.Thread(target=worker)
+  t.daemon = True
+  t.start()
+
 for test_file in test_files:
-  run_test(test_file, verbose)
+  tasks.put(test_file)
+
+tasks.join()
 
 if len(compile_failures) > 0:
-  print
+  print()
   print("Compilation failures:")
   print("=====================\n")
   for f in compile_failures:
     print("    \033[1;31m" + f + "\033[1;m")
 
 if len(run_failures) > 0:
-  print
+  print()
   print("Run failures:")
   print("=============\n")
   for f in run_failures:
     print("    \033[1;33m" + f + "\033[1;m")
 
 print("\n\033[1;32m{}\033[1;m / \033[1;33m{}\033[1;m / {} tests passed"
       .format(run_count, compile_count - run_count,
               run_count + len(compile_failures) + len(run_failures)))