[foozzie] Initial correctness fuzzer harness.

tools/foozzie/v8_foozzie.py

+#!/usr/bin/env python
+# Copyright 2016 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""
+V8 correctness fuzzer launcher script.
+"""
+
+import argparse
+import itertools
+import os
+import re
+import sys
+import traceback
+
+import v8_commands
+import v8_suppressions
+
+CONFIGS = dict(
+  default=[],
+  validate_asm=['--validate-asm'], # Maybe add , '--disable-asm-warnings'
+  fullcode=['--nocrankshaft', '--turbo-filter=~'],
+  noturbo=['--turbo-filter=~', '--noturbo-asm'],
+  noturbo_opt=['--always-opt', '--turbo-filter=~', '--noturbo-asm'],
+  ignition_staging=['--ignition-staging'],
+  ignition_turbo=['--ignition-staging', '--turbo'],
+  ignition_turbo_opt=['--ignition-staging', '--turbo', '--always-opt'],
+)
+
+# Timeout in seconds for one d8 run.
+TIMEOUT = 3
+
+# Return codes.
+RETURN_PASS = 0
+RETURN_FAILURE = 2
+
+BASE_PATH = os.path.dirname(os.path.abspath(__file__))
+PREAMBLE = [
+  os.path.join(BASE_PATH, 'v8_mock.js'),
+  os.path.join(BASE_PATH, 'v8_suppressions.js'),
+]
+
+FLAGS = ['--abort_on_stack_overflow', '--expose-gc', '--allow-natives-syntax',
+         '--invoke-weak-callbacks', '--omit-quit', '--es-staging']
+
+SUPPORTED_ARCHS = ['ia32', 'x64', 'arm', 'arm64']
+
+# Output for suppressed failure case.
+FAILURE_HEADER_TEMPLATE = """
+#
+# V8 correctness failure
+# V8 correctness configs: %(configs)s
+# V8 correctness sources: %(sources)s
+# V8 correctness suppression: %(suppression)s""".strip()
+
+# Extended output for failure case. The 'CHECK' is for the minimizer.
+FAILURE_TEMPLATE = FAILURE_HEADER_TEMPLATE + """
+#
+# CHECK
+#
+# Compared %(first_config_label)s with %(second_config_label)s
+#
+# Flags of %(first_config_label)s:
+%(first_config_flags)s
+# Flags of %(second_config_label)s:
+%(second_config_flags)s
+#
+# Difference:
+%(difference)s
+#
+### Start of configuration %(first_config_label)s:
+%(first_config_output)s
+### End of configuration %(first_config_label)s
+#
+### Start of configuration %(second_config_label)s:
+%(second_config_output)s
+### End of configuration %(second_config_label)s
+""".strip()
+
+
+def parse_args():
+  parser = argparse.ArgumentParser()
+  parser.add_argument(
+    '--random-seed', type=int, required=True,
+    help='random seed passed to both runs')
+  parser.add_argument(
+      '--first-arch', help='first architecture', default='x64')
+  parser.add_argument(
+      '--second-arch', help='second architecture', default='x64')
+  parser.add_argument(
+      '--first-config', help='first configuration', default='fullcode')
+  parser.add_argument(
+      '--second-config', help='second configuration', default='fullcode')
+  parser.add_argument(
+      '--first-d8', default='d8',
+      help='optional path to first d8 executable, '
+           'default: bundled in the same directory as this script')
+  parser.add_argument(
+      '--second-d8',
+      help='optional path to second d8 executable, default: same as first')
+  parser.add_argument('testcase', help='path to test case')
+  options = parser.parse_args()
+
+  # Ensure we make a sane comparison.
+  assert (options.first_arch != options.second_arch or
+          options.first_config != options.second_config) , (
+      'Need either arch or config difference.')
+  assert options.first_arch in SUPPORTED_ARCHS
+  assert options.second_arch in SUPPORTED_ARCHS
+  assert options.first_config in CONFIGS
+  assert options.second_config in CONFIGS
+
+  # Ensure we have a test case.
+  assert (os.path.exists(options.testcase) and
+          os.path.isfile(options.testcase)), (
+      'Test case %s doesn\'t exist' % options.testcase)
+
+  # Use first d8 as default for second d8.
+  options.second_d8 = options.second_d8 or options.first_d8
+
+  # Ensure absolute paths.
+  options.first_d8 = os.path.abspath(options.first_d8)
+  options.second_d8 = os.path.abspath(options.second_d8)
+
+  # Ensure executables exist.
+  assert os.path.exists(options.first_d8)
+  assert os.path.exists(options.second_d8)
+
+  # Ensure we use different executables when we claim we compare
+  # different architectures.
+  # TODO(machenbach): Infer arch from gn's build output.
+  if options.first_arch != options.second_arch:
+    assert options.first_d8 != options.second_d8
+
+  return options
+
+
+def test_pattern_bailout(testcase, ignore_fun):
+  """Print failure state and return if ignore_fun matches testcase."""
+  with open(testcase) as f:
+    bug = (ignore_fun(f.read()) or '').strip()
+    if bug:
+      print FAILURE_HEADER_TEMPLATE % dict(
+          configs='', sources='', suppression=bug)
+      return True
+  return False
+
+
+def pass_bailout(output, step_number):
+  """Print info and return if in timeout or crash pass states."""
+  if output.HasTimedOut():
+    # Dashed output, so that no other clusterfuzz tools can match the
+    # words timeout or crash.
+    print '# V8 correctness - T-I-M-E-O-U-T %d' % step_number
+    return True
+  if output.HasCrashed():
+    print '# V8 correctness - C-R-A-S-H %d' % step_number
+    return True
+  return False
+
+
+def fail_bailout(output, ignore_by_output_fun):
+  """Print failure state and return if ignore_by_output_fun matches output."""
+  bug = (ignore_by_output_fun(output.stdout) or '').strip()
+  if bug:
+    print FAILURE_HEADER_TEMPLATE % dict(
+        configs='', sources='', suppression=bug)
+    return True
+  return False
+
+
+def main():
+  options = parse_args()
+
+  # Suppressions are architecture and configuration specific.
+  suppress = v8_suppressions.get_suppression(
+      options.first_arch, options.first_config,
+      options.second_arch, options.second_config,
+  )
+
+  if test_pattern_bailout(options.testcase, suppress.ignore):
+    return RETURN_FAILURE
+
+  common_flags = FLAGS + ['--random-seed', str(options.random_seed)]
+  first_config_flags = common_flags + CONFIGS[options.first_config]
+  second_config_flags = common_flags + CONFIGS[options.second_config]
+
+  def run_d8(d8, config_flags):
+    return v8_commands.Execute(
+        [d8] + config_flags + PREAMBLE + [options.testcase],
+        cwd=os.path.dirname(options.testcase),
+        timeout=TIMEOUT,
+    )
+
+  first_config_output = run_d8(options.first_d8, first_config_flags)
+
+  # Early bailout based on first run's output.
+  if pass_bailout(first_config_output, 1):
+    return RETURN_PASS
+  if fail_bailout(first_config_output, suppress.ignore_by_output1):
+    return RETURN_FAILURE
+
+  second_config_output = run_d8(options.second_d8, second_config_flags)
+
+  # Bailout based on second run's output.
+  if pass_bailout(second_config_output, 2):
+    return RETURN_PASS
+  if fail_bailout(second_config_output, suppress.ignore_by_output2):
+    return RETURN_FAILURE
+
+  difference = suppress.diff(
+      first_config_output.stdout, second_config_output.stdout)
+  if difference:
+    # The first three entries will be parsed by clusterfuzz. Format changes
+    # will require changes on the clusterfuzz side.
+    first_config_label = '%s,%s' % (options.first_arch, options.first_config)
+    second_config_label = '%s,%s' % (options.second_arch, options.second_config)
+    print FAILURE_TEMPLATE % dict(
+        configs='%s:%s' % (first_config_label, second_config_label),
+        sources='', # TODO
+        suppression='', # We can't tie bugs to differences.
+        first_config_label=first_config_label,
+        second_config_label=second_config_label,
+        first_config_flags=' '.join(first_config_flags),
+        second_config_flags=' '.join(second_config_flags),
+        first_config_output=first_config_output.stdout,
+        second_config_output=second_config_output.stdout,
+        difference=difference,
+    )
+    return RETURN_FAILURE
+
+  # TODO(machenbach): Figure out if we could also return a bug in case there's
+  # no difference, but one of the line suppressions has matched - and without
+  # the match there would be a difference.
+
+  print '# V8 correctness - pass'
+  return RETURN_PASS
+
+
+if __name__ == "__main__":
+  try:
+    result = main()
+  except SystemExit:
+    # Make sure clusterfuzz reports internal errors and wrong usage.
+    # Use one label for all internal and usage errors.
+    print FAILURE_HEADER_TEMPLATE % dict(
+        configs='', sources='', suppression='wrong_usage')
+    result = RETURN_FAILURE
+  except Exception as e:
+    print FAILURE_HEADER_TEMPLATE % dict(
+        configs='', sources='', suppression='internal_error')
+    print '# Internal error: %s' % e
+    traceback.print_exc(file=sys.stdout)
+    result = RETURN_FAILURE
+
+  sys.exit(result)