OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright (c) 2015 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2015 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 | 6 |
7 """Run the given command through LLVM's coverage tools.""" | 7 """Run the given command through LLVM's coverage tools.""" |
8 | 8 |
9 | 9 |
10 import argparse | 10 import argparse |
11 import json | |
12 import os | 11 import os |
13 import re | |
14 import shlex | |
15 import subprocess | 12 import subprocess |
16 import sys | |
17 | 13 |
18 | 14 |
19 BUILDTYPE = 'Coverage' | 15 BUILDTYPE = 'Coverage' |
20 PROFILE_DATA = 'default.profraw' | 16 PROFILE_DATA = 'default.profraw' |
21 PROFILE_DATA_MERGED = 'prof_merged' | 17 PROFILE_DATA_MERGED = 'prof_merged' |
22 SKIA_OUT = 'SKIA_OUT' | 18 SKIA_OUT = 'SKIA_OUT' |
23 | 19 |
24 | 20 |
25 def _fix_filename(filename): | |
26 """Return a filename which we can use to identify the file. | |
27 | |
28 The file paths printed by llvm-cov take the form: | |
29 | |
30 /path/to/repo/out/dir/../../src/filename.cpp | |
31 | |
32 And then they're truncated to 22 characters with leading ellipses: | |
33 | |
34 ...../../src/filename.cpp | |
35 | |
36 This makes it really tough to determine whether the file actually belongs in | |
37 the Skia repo. This function strips out the leading junk so that, if the file | |
38 exists in the repo, the returned string matches the end of some relative path | |
39 in the repo. This doesn't guarantee correctness, but it's about as close as | |
40 we can get. | |
41 """ | |
42 return filename.split('..')[-1].lstrip('./') | |
43 | |
44 | |
45 def _filter_results(results): | |
46 """Filter out any results for files not in the Skia repo. | |
47 | |
48 We run through the list of checked-in files and determine whether each file | |
49 belongs in the repo. Unfortunately, llvm-cov leaves us with fragments of the | |
50 file paths, so we can't guarantee accuracy. See the docstring for | |
51 _fix_filename for more details. | |
52 """ | |
53 all_files = subprocess.check_output(['git', 'ls-files']).splitlines() | |
54 filtered = [] | |
55 for percent, filename in results: | |
56 new_file = _fix_filename(filename) | |
57 matched = [] | |
58 for f in all_files: | |
59 if f.endswith(new_file): | |
60 matched.append(f) | |
61 if len(matched) == 1: | |
62 filtered.append((percent, matched[0])) | |
63 elif len(matched) > 1: | |
64 print >> sys.stderr, ('WARNING: multiple matches for %s; skipping:\n\t%s' | |
65 % (new_file, '\n\t'.join(matched))) | |
66 print 'Filtered out %d files.' % (len(results) - len(filtered)) | |
67 return filtered | |
68 | |
69 | |
70 def _get_out_dir(): | 21 def _get_out_dir(): |
71 """Determine the location for compiled binaries.""" | 22 """Determine the location for compiled binaries.""" |
72 return os.path.join(os.environ.get(SKIA_OUT, os.path.realpath('out')), | 23 return os.path.join(os.environ.get(SKIA_OUT, os.path.realpath('out')), |
73 BUILDTYPE) | 24 BUILDTYPE) |
74 | 25 |
75 | 26 |
76 def run_coverage(cmd): | 27 def run_coverage(cmd): |
77 """Run the given command and return per-file coverage data. | 28 """Run the given command and return per-file coverage data. |
78 | 29 |
79 Assumes that the binary has been built using llvm_coverage_build and that | 30 Assumes that the binary has been built using llvm_coverage_build and that |
80 LLVM 3.6 or newer is installed. | 31 LLVM 3.6 or newer is installed. |
81 """ | 32 """ |
82 binary_path = os.path.join(_get_out_dir(), cmd[0]) | 33 binary_path = os.path.join(_get_out_dir(), cmd[0]) |
83 subprocess.call([binary_path] + cmd[1:]) | 34 subprocess.call([binary_path] + cmd[1:]) |
84 try: | 35 try: |
85 subprocess.check_call( | 36 subprocess.check_call( |
86 ['llvm-profdata', 'merge', PROFILE_DATA, | 37 ['llvm-profdata', 'merge', PROFILE_DATA, |
87 '-output=%s' % PROFILE_DATA_MERGED]) | 38 '-output=%s' % PROFILE_DATA_MERGED]) |
88 finally: | 39 finally: |
89 os.remove(PROFILE_DATA) | 40 os.remove(PROFILE_DATA) |
90 try: | 41 try: |
91 report = subprocess.check_output( | 42 return subprocess.check_output(['llvm-cov', 'show', '-no-colors', |
92 ['llvm-cov', 'report', '-instr-profile', PROFILE_DATA_MERGED, | 43 '-instr-profile', PROFILE_DATA_MERGED, |
93 binary_path]) | 44 binary_path]) |
94 finally: | 45 finally: |
95 os.remove(PROFILE_DATA_MERGED) | 46 os.remove(PROFILE_DATA_MERGED) |
96 results = [] | |
97 for line in report.splitlines()[2:-2]: | |
98 filename, _, _, cover, _, _ = shlex.split(line) | |
99 percent = float(cover.split('%')[0]) | |
100 results.append((percent, filename)) | |
101 results = _filter_results(results) | |
102 results.sort() | |
103 return results | |
104 | |
105 | |
106 def _testname(filename): | |
107 """Transform the file name into an ingestible test name.""" | |
108 return re.sub(r'[^a-zA-Z0-9]', '_', filename) | |
109 | |
110 | |
111 def _nanobench_json(results, properties, key): | |
112 """Return the results in JSON format like that produced by nanobench.""" | |
113 rv = {} | |
114 # Copy over the properties first, then set the 'key' and 'results' keys, | |
115 # in order to avoid bad formatting in case the user passes in a properties | |
116 # dict containing those keys. | |
117 rv.update(properties) | |
118 rv['key'] = key | |
119 rv['results'] = { | |
120 _testname(f): { | |
121 'coverage': { | |
122 'percent': percent, | |
123 'options': { | |
124 'fullname': f, | |
125 'dir': os.path.dirname(f), | |
126 }, | |
127 }, | |
128 } for percent, f in results | |
129 } | |
130 return rv | |
131 | |
132 | |
133 def _parse_key_value(kv_list): | |
134 """Return a dict whose key/value pairs are derived from the given list. | |
135 | |
136 For example: | |
137 | |
138 ['k1', 'v1', 'k2', 'v2'] | |
139 becomes: | |
140 | |
141 {'k1': 'v1', | |
142 'k2': 'v2'} | |
143 """ | |
144 if len(kv_list) % 2 != 0: | |
145 raise Exception('Invalid key/value pairs: %s' % kv_list) | |
146 | |
147 rv = {} | |
148 for i in xrange(len(kv_list) / 2): | |
149 rv[kv_list[i*2]] = kv_list[i*2+1] | |
150 return rv | |
151 | 47 |
152 | 48 |
153 def main(): | 49 def main(): |
154 """Run coverage and generate a report.""" | 50 """Run coverage and generate a report.""" |
155 # Parse args. | 51 # Parse args. |
156 parser = argparse.ArgumentParser() | 52 parser = argparse.ArgumentParser() |
157 parser.add_argument('--outResultsFile') | 53 parser.add_argument('--outResultsFile') |
158 parser.add_argument( | |
159 '--key', metavar='key_or_value', nargs='+', | |
160 help='key/value pairs identifying this bot.') | |
161 parser.add_argument( | |
162 '--properties', metavar='key_or_value', nargs='+', | |
163 help='key/value pairs representing properties of this build.') | |
164 args, cmd = parser.parse_known_args() | 54 args, cmd = parser.parse_known_args() |
165 | 55 |
166 # We still need to pass the args we stripped out to DM. | |
167 cmd.append('--key') | |
168 cmd.extend(args.key) | |
169 cmd.append('--properties') | |
170 cmd.extend(args.properties) | |
171 | |
172 # Parse the key and properties for use in the nanobench JSON output. | |
173 key = _parse_key_value(args.key) | |
174 properties = _parse_key_value(args.properties) | |
175 | |
176 # Run coverage. | 56 # Run coverage. |
177 results = run_coverage(cmd) | 57 report = run_coverage(cmd) |
178 | 58 with open(args.outResultsFile, 'w') as f: |
179 # Write results. | 59 f.write(report) |
180 format_results = _nanobench_json(results, properties, key) | |
181 if args.outResultsFile: | |
182 with open(args.outResultsFile, 'w') as f: | |
183 json.dump(format_results, f) | |
184 else: | |
185 print json.dumps(format_results, indent=4, sort_keys=True) | |
186 | 60 |
187 | 61 |
188 if __name__ == '__main__': | 62 if __name__ == '__main__': |
189 main() | 63 main() |
OLD | NEW |