Rewrite blink_gc_plugin test script in Python.

tools/clang/blink_gc_plugin/tests/test.py

Index: tools/clang/blink_gc_plugin/tests/test.py
diff --git a/tools/clang/blink_gc_plugin/tests/test.py b/tools/clang/blink_gc_plugin/tests/test.py
new file mode 100755
index 0000000000000000000000000000000000000000..0a97ebc524e12528efdedb340319a7ff760870ea
--- /dev/null
+++ b/tools/clang/blink_gc_plugin/tests/test.py
@@ -0,0 +1,98 @@
+#!/usr/bin/env python
+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+import glob
+import os
+import subprocess
+import sys
+
+
+def run_test(test_base_name, cmd):

[Nico, 2015/05/13 17:43:58]

add docstring that documents what return value means

[dcheng, 2015/05/13 20:59:34]

done

+  try:
+    actual = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+  except subprocess.CalledProcessError as e:
+    # Some of the Blink GC plugin tests intentionally trigger compile errors, so
+    # just ignore an exit code that indicates failure.
+    actual = e.output
+  except Exception as e:
+    return 'could not execute %s (%s)' % (cmd, e)
+
+  # Some Blink GC plugins dump a JSON representation of the object graph, and
+  # use the processed results as the actual results of the test.
+  if 'dump-graph' in cmd:

[Nico, 2015/05/13 17:43:58]

Why check this instead of checking for the existence of the json file like the
sh script does?

[dcheng, 2015/05/13 20:59:33]

I guess I feel weird about checking something and then using it later, due to
potential TOCTOU issues. I guess it doesn't matter here, but this does seem
vaguely nicer to me (because we could hypothetically differentiate between
'dump-graph was present but the json was not generated' and 'the results of
processing the json returned nothing'

I don't care strongly, so I just changed it to match the .sh.

+    try:
+      actual = subprocess.check_output(
+          ['python', '../process-graph.py', '-c',
+           '%s.graph.json' % test_base_name],
+          stderr=subprocess.STDOUT)
+    except subprocess.CalledProcessError, e:
+      # The graph processing script returns a failure exit code if the graph is
+      # 'bad' (e.g. it has a cycle). Hope that the output returned contains the

[Nico, 2015/05/13 17:43:58]

s/Hope/something else/

[dcheng, 2015/05/13 20:59:34]

Done.

+      # right info.
+      actual = e.output
+
+  try:
+    expected = open('%s.txt' % test_base_name).read()

[Nico, 2015/05/13 17:43:58]

Maybe this only needs an [:-1] to strip the newline?

[dcheng, 2015/05/13 20:59:33]

I just added rstrip() to hack around this for now. I think we should eventually
just rebaseline the expected results.

+  except IOError:
+    open('%s.txt' % test_base_name, 'w').write(actual)
+    return 'no expected file found'
+
+  if expected != actual:
+    open('%s.txt' % test_base_name, 'w').write(actual)

[Nico, 2015/05/13 17:43:58]

This overwrites the input, is that intentional?

[dcheng, 2015/05/13 20:59:34]

I wanted to experiment with something better than:
diff foo.txt*
mv foo.txt-actual foo.txt

But I'm not sure how I feel about the "tests automatically start passing if you
run them more than once" bit.

I've reverted it to the original behavior for now. I guess the best thing to do
would be to provide a --reset-results like Blink's layout test runner.

+    return 'expected and actual differed'
+
+
+def run_tests(clang_path, plugin_path):
+  passing = []
+  failing = []
+
+  base_cmd = [clang_path, '-fsyntax-only', '-std=c++11']
+  base_cmd.extend(['-Wno-inaccessible-base'])
+  if plugin_path:
+    base_cmd.extend(['-Xclang', '-load', '-Xclang', plugin_path])
+  base_cmd.extend(['-Xclang', '-add-plugin', '-Xclang', 'blink-gc-plugin'])
+
+  tests = glob.glob('*.cpp')
+  for test in tests:
+    sys.stdout.write('Testing %s... ' % test)
+    test_base_name, _ = os.path.splitext(test)
+
+    cmd = base_cmd[:]
+    try:
+      cmd.extend(file('%s.flags' % test_base_name).read().split())
+    except IOError:
+      pass
+    cmd.append(test)
+
+    failure_message = run_test(test_base_name, cmd)
+    if failure_message:
+      print 'failed: %s' % failure_message
+      failing.append(test_base_name)
+    else:
+      print 'passed!'
+      passing.append(test_base_name)
+
+  return passing, failing
+
+
+def main():
+  if len(sys.argv) < 2:
+    print 'Usage: <path to clang>[ <path to plugin>]'
+    return -1
+
+  os.chdir(os.path.dirname(os.path.realpath(__file__)))
+
+  clang_path = sys.argv[1]
+  plugin_path = sys.argv[2] if len(sys.argv) > 2 else None
+  passing, failing = run_tests(clang_path, plugin_path)
+  print 'Ran %d tests: %d succeeded, %d failed' % (
+      len(passing) + len(failing), len(passing), len(failing))
+  for test in failing:
+    print '    %s' % test
+
+
+if __name__ == '__main__':
+  sys.exit(main())