Add scripts for running LLVM coverage

tools/llvm_coverage_run.py

Index: tools/llvm_coverage_run.py
diff --git a/tools/llvm_coverage_run.py b/tools/llvm_coverage_run.py
new file mode 100755
index 0000000000000000000000000000000000000000..902b1c6b04f1833dc3b6d48c5bfa919f3311bb16
--- /dev/null
+++ b/tools/llvm_coverage_run.py
@@ -0,0 +1,107 @@
+#!/usr/bin/env python
+# Copyright (c) 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+"""Run the given command through LLVM's coverage tools."""
+
+
+import argparse
+import json
+import os
+import shlex
+import subprocess
+import sys
+
+
+BUILDTYPE = 'Coverage'
+OUT_DIR = os.path.realpath(os.path.join('out', BUILDTYPE))
+PROFILE_DATA = 'default.profraw'
+PROFILE_DATA_MERGED = 'prof_merged'
+
+
+def _fix_filename(filename):
+  """Return a filename which we can use to identify the file.
+
+  The file paths printed by llvm-cov take the form:
+
+      /path/to/repo/out/dir/../../src/filename.cpp
+
+  And then they're truncated to 22 characters with leading ellipses:
+
+      ...../../src/filename.cpp
+
+  This makes it really tough to determine whether the file actually belongs in
+  the Skia repo.  This function strips out the leading junk so that, if the file
+  exists in the repo, the returned string matches the end of some relative path
+  in the repo. This doesn't guarantee correctness, but it's about as close as
+  we can get.
+  """
+  return filename.split('..')[-1].lstrip('./')
+
+
+def _filter_results(results):
+  """Filter out any results for files not in the Skia repo.
+
+  We run through the list of checked-in files and determine whether each file
+  belongs in the repo. Unfortunately, llvm-cov leaves us with fragments of the
+  file paths, so we can't guarantee accuracy. See the docstring for
+  _fix_filename for more details.
+  """
+  all_files = subprocess.check_output(['git', 'ls-files']).splitlines()
+  filtered = []
+  for percent, filename in results:
+    new_file = _fix_filename(filename)
+    matched = []
+    for f in all_files:
+      if f.endswith(new_file):
+        matched.append(f)
+    if len(matched) == 1:
+      filtered.append((percent, matched[0]))
+    elif len(matched) > 1:
+      print >> sys.stderr, ('WARNING: multiple matches for %s; skipping:\n\t%s'
+                            % (new_file, '\n\t'.join(matched)))
+  print 'Filtered out %d files.' % (len(results) - len(filtered))
+  return filtered

[borenet, 2015/07/01 20:06:51]

The right solution here is to submit a change to LLVM which adds the option to
print the full path to the file.  For the moment, just skip the files we can't
find.

[mtklein, 2015/07/01 20:17:37]

I wonder how basename.startswith('Sk') works as our test.

[borenet, 2015/07/06 12:06:35]

Maybe.  The biggest problem with that is we'll miss all of the files whose names
are longer than 22 characters.

+
+
+def run_coverage(cmd):
+  """Run the given command and return per-file coverage data.
+
+  Assumes that the binary has been built using llvm_coverage_build and that
+  LLVM 3.6 or newer is installed.
+  """
+  binary_path = os.path.join(OUT_DIR, cmd[0])
+  subprocess.call([binary_path] + cmd[1:])
+  try:
+    subprocess.check_call(
+        ['llvm-profdata', 'merge', PROFILE_DATA,
+         '-output=%s' % PROFILE_DATA_MERGED])
+  finally:
+    os.remove(PROFILE_DATA)
+  try:
+    report = subprocess.check_output(
+        ['llvm-cov', 'report', '-instr-profile', PROFILE_DATA_MERGED,
+         binary_path])
+  finally:
+    os.remove(PROFILE_DATA_MERGED)
+  results = []
+  for line in report.splitlines()[2:-2]:
+    filename, _, _, cover, _, _ = shlex.split(line)
+    percent = float(cover.split('%')[0])
+    results.append((percent, filename))
+  results = _filter_results(results)
+  results.sort()
+  return results
+
+
+def main():
+  res = run_coverage(sys.argv[1:])
+  print '% Covered\tFilename'
+  for percent, f in res:

[mtklein, 2015/07/01 20:17:37]

Can we get a line-by-line breakdown or some sort of explorable HTML thing?

[borenet, 2015/07/06 12:06:35]

`llvm-cov show` generates a giant file which shows the code annotated with the
number of times each region was hit.  Unfortunately, it's a single file with
everything, and it's not nearly as nice to navigate as the HTML report generated
by genhtml.  I'd assumed I could find an equivalent but I haven't so far.

+    print '%f\t%s' % (percent, f)
+
+
+if __name__ == '__main__':
+  main()