OLD | NEW |
---|---|
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright 2015 The Chromium Authors. All rights reserved. | 2 # Copyright 2015 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 | 6 |
7 import argparse | |
7 import glob | 8 import glob |
8 import os | 9 import os |
9 import subprocess | 10 import subprocess |
10 import sys | 11 import sys |
11 | 12 |
12 | 13 |
13 def run_test(test_base_name, cmd): | 14 def run_test(test_base_name, cmd, reset_results): |
14 """Run a test case. | 15 """Run a test case. |
15 | 16 |
16 Args: | 17 Args: |
17 test_base_name: The name for the test C++ source file without the extension. | 18 test_base_name: The name for the test C++ source file without the extension. |
18 cmd: The actual command to run for the test. | 19 cmd: The actual command to run for the test. |
20 reset_results: True if the results should be overwritten in place. | |
19 | 21 |
20 Returns: | 22 Returns: |
21 None on pass, or a str with the description of the failure. | 23 None on pass, or a str with the description of the failure. |
22 """ | 24 """ |
23 try: | 25 try: |
24 actual = subprocess.check_output(cmd, stderr=subprocess.STDOUT) | 26 actual = subprocess.check_output(cmd, stderr=subprocess.STDOUT) |
25 except subprocess.CalledProcessError as e: | 27 except subprocess.CalledProcessError as e: |
26 # Some of the Blink GC plugin tests intentionally trigger compile errors, so | 28 # Some of the Blink GC plugin tests intentionally trigger compile errors, so |
27 # just ignore an exit code that indicates failure. | 29 # just ignore an exit code that indicates failure. |
28 actual = e.output | 30 actual = e.output |
(...skipping 11 matching lines...) Expand all Loading... | |
40 except subprocess.CalledProcessError, e: | 42 except subprocess.CalledProcessError, e: |
41 # The graph processing script returns a failure exit code if the graph is | 43 # The graph processing script returns a failure exit code if the graph is |
42 # 'bad' (e.g. it has a cycle). The output still needs to be captured in | 44 # 'bad' (e.g. it has a cycle). The output still needs to be captured in |
43 # that case, since the expected results capture the errors. | 45 # that case, since the expected results capture the errors. |
44 actual = e.output | 46 actual = e.output |
45 finally: | 47 finally: |
46 # Clean up the .graph.json file to prevent false passes from stale results | 48 # Clean up the .graph.json file to prevent false passes from stale results |
47 # from a previous run. | 49 # from a previous run. |
48 os.remove('%s.graph.json' % test_base_name) | 50 os.remove('%s.graph.json' % test_base_name) |
49 | 51 |
50 # TODO(dcheng): Remove the rstrip() and just rebaseline the tests to match. | |
51 actual = actual.rstrip() | |
52 | |
53 # On Windows, clang emits CRLF as the end of line marker. Normalize it to LF | 52 # On Windows, clang emits CRLF as the end of line marker. Normalize it to LF |
54 # to match posix systems. | 53 # to match posix systems. |
55 actual = actual.replace('\r\n', '\n') | 54 actual = actual.replace('\r\n', '\n') |
56 | 55 |
56 result_file = '%s.txt%s' % ( | |
57 test_base_name, '' if reset_results else '.actual') | |
57 try: | 58 try: |
58 expected = open('%s.txt' % test_base_name).read().rstrip() | 59 expected = open('%s.txt' % test_base_name).read() |
59 except IOError: | 60 except IOError: |
60 open('%s.txt.actual' % test_base_name, 'w').write(actual) | 61 open(result_file, 'w').write(actual) |
61 return 'no expected file found' | 62 return 'no expected file found' |
62 | 63 |
63 if expected != actual: | 64 if expected != actual: |
64 open('%s.txt.actual' % test_base_name, 'w').write(actual) | 65 open(result_file, 'w').write(actual) |
65 return 'expected and actual differed' | 66 return 'expected and actual differed' |
66 | 67 |
67 | 68 |
68 def run_tests(clang_path, plugin_path): | 69 def run_tests(clang_path, plugin_path, reset_results): |
69 """Runs the tests. | 70 """Runs the tests. |
70 | 71 |
71 Args: | 72 Args: |
72 clang_path: The path to the clang binary to be tested. | 73 clang_path: The path to the clang binary to be tested. |
73 plugin_path: An optional path to the plugin to test. This may be None, if | 74 plugin_path: An optional path to the plugin to test. This may be None, if |
74 plugin is built directly into clang, like on Windows. | 75 plugin is built directly into clang, like on Windows. |
76 reset_results: True if the results should be overwritten in place. | |
75 | 77 |
76 Returns: | 78 Returns: |
77 (passing, failing): Two lists containing the base names of the passing and | 79 (passing, failing): Two lists containing the base names of the passing and |
78 failing tests respectively. | 80 failing tests respectively. |
79 """ | 81 """ |
80 passing = [] | 82 passing = [] |
81 failing = [] | 83 failing = [] |
82 | 84 |
83 # The plugin option to dump the object graph is incompatible with | 85 # The plugin option to dump the object graph is incompatible with |
84 # -fsyntax-only. It generates the .graph.json file based on the name of the | 86 # -fsyntax-only. It generates the .graph.json file based on the name of the |
85 # output file, but there is no output filename with -fsyntax-only. | 87 # output file, but there is no output filename with -fsyntax-only. |
86 base_cmd = [clang_path, '-c', '-std=c++11'] | 88 base_cmd = [clang_path, '-c', '-std=c++11'] |
87 base_cmd.extend(['-Wno-inaccessible-base']) | 89 base_cmd.extend(['-Wno-inaccessible-base']) |
88 if plugin_path: | 90 if plugin_path: |
89 base_cmd.extend(['-Xclang', '-load', '-Xclang', plugin_path]) | 91 base_cmd.extend(['-Xclang', '-load', '-Xclang', plugin_path]) |
90 base_cmd.extend(['-Xclang', '-add-plugin', '-Xclang', 'blink-gc-plugin']) | 92 base_cmd.extend(['-Xclang', '-add-plugin', '-Xclang', 'blink-gc-plugin']) |
91 | 93 |
92 tests = glob.glob('*.cpp') | 94 tests = glob.glob('*.cpp') |
93 for test in tests: | 95 for test in tests: |
94 sys.stdout.write('Testing %s... ' % test) | 96 sys.stdout.write('Testing %s... ' % test) |
95 test_base_name, _ = os.path.splitext(test) | 97 test_base_name, _ = os.path.splitext(test) |
96 | 98 |
97 cmd = base_cmd[:] | 99 cmd = base_cmd[:] |
98 try: | 100 try: |
99 cmd.extend(file('%s.flags' % test_base_name).read().split()) | 101 cmd.extend(file('%s.flags' % test_base_name).read().split()) |
100 except IOError: | 102 except IOError: |
101 pass | 103 pass |
102 cmd.append(test) | 104 cmd.append(test) |
103 | 105 |
104 failure_message = run_test(test_base_name, cmd) | 106 failure_message = run_test(test_base_name, cmd, reset_results) |
105 if failure_message: | 107 if failure_message: |
106 print 'failed: %s' % failure_message | 108 print 'failed: %s' % failure_message |
107 failing.append(test_base_name) | 109 failing.append(test_base_name) |
108 else: | 110 else: |
109 print 'passed!' | 111 print 'passed!' |
110 passing.append(test_base_name) | 112 passing.append(test_base_name) |
111 | 113 |
112 return passing, failing | 114 return passing, failing |
113 | 115 |
114 | 116 |
115 def main(): | 117 def main(): |
116 if len(sys.argv) < 2: | 118 parser = argparse.ArgumentParser() |
117 print 'Usage: <path to clang>[ <path to plugin>]' | 119 parser.add_argument( |
118 return -1 | 120 '--reset-results', action='store_true', |
121 help='If specified, overwrites the expected results in place.') | |
Nico
2015/05/14 19:29:37
(fwiw, I always thought the webkit baseline test r
| |
122 parser.add_argument('clang_path', help='The path to the clang binary.') | |
123 parser.add_argument('plugin_path', nargs='?', | |
124 help='The path to the plugin library, if any.') | |
125 args = parser.parse_args() | |
119 | 126 |
120 os.chdir(os.path.dirname(os.path.realpath(__file__))) | 127 os.chdir(os.path.dirname(os.path.realpath(__file__))) |
121 | 128 |
122 clang_path = sys.argv[1] | 129 print 'Using clang %s...' % args.clang_path |
123 plugin_path = sys.argv[2] if len(sys.argv) > 2 else None | 130 print 'Using plugin %s...' % args.plugin_path |
124 print 'Using clang %s...' % clang_path | |
125 print 'Using plugin %s...' % plugin_path | |
126 | 131 |
127 passing, failing = run_tests(clang_path, plugin_path) | 132 passing, failing = run_tests(args.clang_path, |
133 args.plugin_path, | |
134 args.reset_results) | |
128 print 'Ran %d tests: %d succeeded, %d failed' % ( | 135 print 'Ran %d tests: %d succeeded, %d failed' % ( |
129 len(passing) + len(failing), len(passing), len(failing)) | 136 len(passing) + len(failing), len(passing), len(failing)) |
130 for test in failing: | 137 for test in failing: |
131 print ' %s' % test | 138 print ' %s' % test |
132 return len(failing) | 139 return len(failing) |
133 | 140 |
134 | 141 |
135 if __name__ == '__main__': | 142 if __name__ == '__main__': |
136 sys.exit(main()) | 143 sys.exit(main()) |
OLD | NEW |