[foozzie] Initial correctness fuzzer harness.

tools/foozzie/v8_foozzie.py

+#!/usr/bin/env python
+# Copyright 2016 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""
+V8 correctness fuzzer launcher script.
+"""
+
+import argparse
+import itertools
+import os
+import re
+import sys
+import traceback
+
+import v8_commands
+import v8_suppressions
+
+CONFIGS = {
+  'default': [],
+  'validate_asm': ['--validate-asm'], # Maybe add , '--disable-asm-warnings'
+  'fullcode': ['--nocrankshaft', '--turbo-filter=~'],
+  'noturbo': ['--turbo-filter=~', '--noturbo-asm'],
+  'noturbo_opt': ['--always-opt', '--turbo-filter=~', '--noturbo-asm'],
+  'ignition_staging': ['--ignition-staging'],
+  'ignition_turbo': ['--ignition-staging', '--turbo'],
+  'ignition_turbo_opt': ['--ignition-staging', '--turbo', '--always-opt'],
+}
+
+# Timeout in seconds for one d8 run.
+TIMEOUT = 3
+
+BASE_PATH = os.path.dirname(os.path.abspath(__file__))
+PREAMBLE = [
+  os.path.join(BASE_PATH, 'v8_mock.js'),
+  os.path.join(BASE_PATH, 'v8_suppressions.js'),
+]
+
+FLAGS = ['--abort_on_stack_overflow', '--expose-gc', '--allow-natives-syntax',
+         '--invoke-weak-callbacks', '--omit-quit', '--es-staging']
+
+SUPPORTED_ARCHS = ['ia32', 'x64', 'arm', 'arm64']
+
+# Output for suppressed failure case.
+FAILURE_HEADER_TEMPLATE = """#

[tandrii(chromium), 2016/12/16 16:46:26]

Nit: instead of """# i'd do:
"""
# 
....
""".strip()

[Michael Achenbach, 2016/12/19 08:42:45]

Done.

+# V8 correctness failure
+# V8 correctness configs: %(configs)s
+# V8 correctness sources: %(sources)s
+# V8 correctness suppression: %(suppression)s"""
+
+# Extended output for failure case. The 'CHECK' is for the minimizer.
+FAILURE_TEMPLATE = FAILURE_HEADER_TEMPLATE + """#
+# CHECK
+#
+# Compared %(first_config_label)s with %(second_config_label)s
+#
+# Flags of %(first_config_label)s:
+%(first_config_flags)s
+# Flags of %(second_config_label)s:
+%(second_config_flags)s
+#
+# Difference:
+%(difference)s
+#
+### Start of configuration %(first_config_label)s:
+%(first_config_output)s
+### End of configuration %(first_config_label)s
+#
+### Start of configuration %(second_config_label)s:
+%(second_config_output)s
+### End of configuration %(second_config_label)s
+"""
+
+
+def parse_args():
+  parser = argparse.ArgumentParser()
+  parser.add_argument(
+    '--random-seed', type=int, required=True,
+    help='random seed passed to both runs')
+  parser.add_argument(
+      '--first-arch', help='first architecture', default='x64')
+  parser.add_argument(
+      '--second-arch', help='second architecture', default='x64')
+  parser.add_argument(
+      '--first-config', help='first configuration', default='fullcode')
+  parser.add_argument(
+      '--second-config', help='second configuration', default='fullcode')
+  parser.add_argument(
+      '--first-d8', default='d8',
+      help='optional path to first d8 executable, default: side-by-side')

[tandrii(chromium), 2016/12/16 16:46:26]

what does "side-by-side" mean?

[Michael Achenbach, 2016/12/19 08:42:45]

Clarified.

+  parser.add_argument(
+      '--second-d8',
+      help='optional path to second d8 executable, default: same as first')
+  parser.add_argument('testcase', help='path to test case')
+  options = parser.parse_args()
+
+  # Ensure we make a sane comparison.
+  assert (options.first_arch != options.second_arch or
+          options.first_config != options.second_config) , (
+      'Need either arch or config difference.')
+  assert options.first_arch in SUPPORTED_ARCHS
+  assert options.second_arch in SUPPORTED_ARCHS
+  assert options.first_config in CONFIGS
+  assert options.second_config in CONFIGS
+
+  # Ensure we have a test case.
+  assert (os.path.exists(options.testcase) and

[tandrii(chromium), 2016/12/16 16:46:26]

if x:
  parser.error()
is nicer, but assert is certainly shorter and easier to read. I suppose this
script isn't supposed to be called manually frequently, so I think assert is
fine.

[Michael Achenbach, 2016/12/19 08:42:45]

The script is designed to be called from automated tooling. parser.error()
actually throws SystemExit, while assert throws assertion error.

+          os.path.isfile(options.testcase)), (
+      'Test case %s doesn\'t exist' % options.testcase)
+
+  # Use first d8 as default for second d8.
+  options.second_d8 = options.second_d8 or options.first_d8
+
+  # Ensure absolute paths.
+  options.first_d8 = os.path.abspath(options.first_d8)
+  options.second_d8 = os.path.abspath(options.second_d8)
+
+  # Ensure executables exist.
+  assert os.path.exists(options.first_d8)
+  assert os.path.exists(options.second_d8)
+
+  # Ensure we use different executables when we claim we compare
+  # different architectures.
+  # TODO(machenbach): Infer arch from gn's build output.
+  if options.first_arch != options.second_arch:
+    assert options.first_d8 != options.second_d8
+
+  return options
+
+
+def test_pattern_bailout(testcase, ignore_fun):
+  """Print failure state and return if ignore_fun matches testcase."""
+  with open(testcase) as f:
+    bug = (ignore_fun(f.read()) or '').strip()
+    if bug:
+      print FAILURE_HEADER_TEMPLATE % {
+        'configs': '',
+        'sources': '',
+        'suppression': bug,
+      }
+      return True
+  return False
+
+
+def pass_bailout(output, step_number):
+  """Print info and return if in timeout or crash pass states."""
+  if output.HasTimedOut():
+    # Dashed output, so that no other clusterfuzz tools can match the
+    # words timeout or crash.
+    print '# V8 correctness - T-I-M-E-O-U-T %d' % step_number
+    return True
+  if output.HasCrashed():
+    print '# V8 correctness - C-R-A-S-H %d' % step_number
+    return True
+  return False
+
+
+def fail_bailout(output, ignore_by_output_fun):
+  """Print failure state and return if ignore_by_output_fun matches output."""
+  bug = (ignore_by_output_fun(output.stdout) or '').strip()
+  if bug:
+    print FAILURE_HEADER_TEMPLATE % {
+      'configs': '',
+      'sources': '',
+      'suppression': bug,
+    }
+    return True
+  return False
+
+
+def main():
+  options = parse_args()
+
+  # Suppressions are architecture and configuration specific.
+  suppress = v8_suppressions.get_suppression(
+      options.first_arch, options.first_config,
+      options.second_arch, options.second_config,
+  )
+
+  if test_pattern_bailout(options.testcase, suppress.ignore):
+    return 2

[tandrii(chromium), 2016/12/16 16:46:26]

i'd make 2 a meaningful constant, perhaps 0 too.

[Michael Achenbach, 2016/12/19 08:42:45]

Done.

+
+  common_flags = FLAGS + ['--random-seed', str(options.random_seed)]
+  first_config_flags = common_flags + CONFIGS[options.first_config]
+  second_config_flags = common_flags + CONFIGS[options.second_config]
+
+  def run_d8(d8, config_flags):
+    return v8_commands.Execute(
+        [d8] + config_flags + PREAMBLE + [options.testcase],
+        cwd=os.path.dirname(options.testcase),
+        timeout=TIMEOUT,
+    )
+
+  first_config_output = run_d8(options.first_d8, first_config_flags)
+
+  # Early bailout based on first run's output.
+  if pass_bailout(first_config_output, 1):
+    return 0
+  if fail_bailout(first_config_output, suppress.ignore_by_output1):
+    return 2
+
+  second_config_output = run_d8(options.second_d8, second_config_flags)
+
+  # Bailout based on second run's output.
+  if pass_bailout(second_config_output, 2):
+    return 0
+  if fail_bailout(second_config_output, suppress.ignore_by_output2):
+    return 2
+
+  difference = suppress.diff(
+      first_config_output.stdout, second_config_output.stdout)
+  if difference:
+    # The first three entries will be parsed by clusterfuzz. Format changes
+    # will require changes on the clusterfuzz side.
+    first_config_label = '%s,%s' % (options.first_arch, options.first_config)
+    second_config_label = '%s,%s' % (options.second_arch, options.second_config)
+    print FAILURE_TEMPLATE % {
+      'configs': '%s:%s' % (first_config_label, second_config_label),
+      'sources': '', # TODO
+      'suppression': '', # We can't tie bugs to differences.
+      'first_config_label': first_config_label,
+      'second_config_label': second_config_label,
+      'first_config_flags': ' '.join(first_config_flags),
+      'second_config_flags': ' '.join(second_config_flags),
+      'first_config_output': first_config_output.stdout,
+      'second_config_output': second_config_output.stdout,
+      'difference': difference,
+    }
+    return 2
+
+  # TODO(machenbach): Figure out if we could also return a bug in case there's
+  # no difference, but one of the line suppressions has matched - and without
+  # the match there would be a difference.
+
+  print '# V8 correctness - pass'
+  return 0
+
+
+if __name__ == "__main__":
+  try:
+    result = main()
+  except SystemExit:
+    # Make sure clusterfuzz reports internal errors and wrong usage.
+    # Use one label for all internal and usage errors.
+    print FAILURE_HEADER_TEMPLATE % {
+      'configs': '',

[tandrii(chromium), 2016/12/19 08:43:32]

nitty IMO: % dict(configs='', sources='', suppression='')
it's handy if one's editor checks variables usage and highlights likely
misspelled ones.

[Michael Achenbach, 2016/12/19 09:42:19]

Done. Also for the other dicts here.

+      'sources': '',
+      'suppression': 'wrong_usage',
+    }
+    result = 2
+  except Exception as e:
+    print FAILURE_HEADER_TEMPLATE % {
+      'configs': '',
+      'sources': '',
+      'suppression': 'internal_error',
+    }
+    print '# Internal error: %s' % e
+    traceback.print_exc(file=sys.stdout)
+    result = 2
+
+  sys.exit(result)
+