Add croc code coverage tool. (Same change as yesterday, but now made in the...

tools/code_coverage/croc.py

+#!/usr/bin/python2.4
+#
+# Copyright 2009, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#        * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#        * Redistributions in binary form must reproduce the above
+#     copyright notice, this list of conditions and the following disclaimer
+#     in the documentation and/or other materials provided with the
+#     distribution.
+#        * Neither the name of Google Inc. nor the names of its
+#     contributors may be used to endorse or promote products derived from
+#     this software without specific prior written permission.
+#
+#     THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#     "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#     LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#     A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#     OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#     SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#     LIMITED TO, PROCUREMENT  OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#     DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#     THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#     (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#     OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Crocodile - compute coverage numbers for Chrome coverage dashboard."""
+
+import os
+import re
+import sys
+from optparse import OptionParser
+
+
+class CoverageError(Exception):
+  """Coverage error."""
+
+class CoverageStatError(CoverageError):
+  """Error evaluating coverage stat."""
+
+#------------------------------------------------------------------------------
+
+
+class CoverageStats(dict):
+  """Coverage statistics."""
+
+  def Add(self, coverage_stats):
+    """Adds a contribution from another coverage stats dict.
+
+    Args:
+      coverage_stats: Statistics to add to this one.
+    """
+    for k, v in coverage_stats.iteritems():
+      if k in self:
+        self[k] = self[k] + v
+      else:
+        self[k] = v
+
+#------------------------------------------------------------------------------
+
+
+class CoveredFile(object):
+  """Information about a single covered file."""
+
+  def __init__(self, filename, group, language):
+    """Constructor.
+
+    Args:
+      filename: Full path to file, '/'-delimited.
+      group: Group file belongs to.
+      language: Language for file.
+    """
+    self.filename = filename
+    self.group = group
+    self.language = language
+
+    # No coverage data for file yet
+    self.lines = {}     # line_no -> None=executable, 0=instrumented, 1=covered
+    self.stats = CoverageStats()
+
+  def UpdateCoverage(self):
+    """Updates the coverage summary based on covered lines."""
+    exe = instr = cov = 0
+    for l in self.lines.itervalues():
+      exe += 1
+      if l is not None:
+        instr += 1
+        if l == 1:
+          cov += 1
+
+    # Add stats that always exist
+    self.stats = CoverageStats(lines_executable=exe,
+                               lines_instrumented=instr,
+                               lines_covered=cov,
+                               files_executable=1)
+
+    # Add conditional stats
+    if cov:
+      self.stats['files_covered'] = 1
+    if instr:
+      self.stats['files_instrumented'] = 1
+
+
+#------------------------------------------------------------------------------
+
+
+class CoveredDir(object):
+  """Information about a directory containing covered files."""
+
+  def __init__(self, dirpath):
+    """Constructor.
+
+    Args:
+      dirpath: Full path of directory, '/'-delimited.
+    """
+    self.dirpath = dirpath
+
+    # List of covered files directly in this dir, indexed by filename (not
+    # full path)
+    self.files = {}
+
+    # List of subdirs, indexed by filename (not full path)
+    self.subdirs = {}
+
+    # Dict of CoverageStats objects summarizing all children, indexed by group
+    self.stats_by_group = {'all':CoverageStats()}
+    # TODO: by language
+
+  def GetTree(self, indent=''):
+    """Recursively gets stats for the directory and its children.
+
+    Args:
+      indent: indent prefix string.
+
+    Returns:
+      The tree as a string.
+    """
+    dest = []
+
+    # Compile all groupstats
+    groupstats = []
+    for group in sorted(self.stats_by_group):
+      s = self.stats_by_group[group]
+      if not s.get('lines_executable'):
+        continue        # Skip groups with no executable lines
+      groupstats.append('%s:%d/%d/%d' % (
+          group, s.get('lines_covered', 0),
+          s.get('lines_instrumented', 0),
+          s.get('lines_executable', 0)))
+
+    outline = '%s%-30s   %s' % (indent,
+                                self.dirpath + '/', '   '.join(groupstats))
+    dest.append(outline.rstrip())
+
+    for d in sorted(self.subdirs):
+      dest.append(self.subdirs[d].GetTree(indent=indent + '  '))
+
+    return '\n'.join(dest)
+
+#------------------------------------------------------------------------------
+
+
+class Coverage(object):
+  """Code coverage for a group of files."""
+
+  def __init__(self):
+    """Constructor."""
+    self.files = {}             # Map filename --> CoverageFile
+    self.root_dirs = []         # (root, altname)
+    self.rules = []             # (regexp, include, group, language)
+    self.tree = CoveredDir('')
+    self.print_stats = []       # Dicts of args to PrintStat()
+
+    self.add_files_walk = os.walk       # Walk function for AddFiles()
+
+    # Must specify subdir rule, or AddFiles() won't find any files because it
+    # will prune out all the subdirs.  Since subdirs never match any code,
+    # they won't be reported in other stats, so this is ok.
+    self.AddRule('.*/$', language='subdir')
+
+
+  def CleanupFilename(self, filename):
+    """Cleans up a filename.
+
+    Args:
+      filename: Input filename.
+
+    Returns:
+      The cleaned up filename.
+
+    Changes all path separators to '/'.
+    Makes relative paths (those starting with '../' or './' absolute.
+    Replaces all instances of root dirs with alternate names.
+    """
+    # Change path separators
+    filename = filename.replace('\\', '/')
+
+    # If path is relative, make it absolute
+    # TODO: Perhaps we should default to relative instead, and only understand
+    # absolute to be files starting with '\', '/', or '[A-Za-z]:'?
+    if filename.split('/')[0] in ('.', '..'):
+      filename = os.path.abspath(filename).replace('\\', '/')
+
+    # Replace alternate roots
+    for root, alt_name in self.root_dirs:
+        filename = re.sub('^' + re.escape(root) + '(?=(/|$))',
+                          alt_name, filename)
+    return filename
+
+  def ClassifyFile(self, filename):
+    """Applies rules to a filename, to see if we care about it.
+
+    Args:
+      filename: Input filename.
+
+    Returns:
+      (None, None) if the file is not included or has no group or has no
+          language.  Otherwise, a 2-tuple containing:
+      The group for the file (for example, 'source' or 'test').
+      The language of the file.
+    """
+    include = False
+    group = None
+    language = None
+
+    # Process all rules
+    for regexp, rule_include, rule_group, rule_language in self.rules:
+      if regexp.match(filename):
+        # include/exclude source
+        if rule_include is not None:
+          include = rule_include
+        if rule_group is not None:
+          group = rule_group
+        if rule_language is not None:
+          language = rule_language
+
+    # TODO: Should have a debug mode which prints files which aren't excluded
+    # and why (explicitly excluded, no type, no language, etc.)
+
+    # TODO: Files can belong to multiple groups?
+    #   (test/source)
+    #   (mac/pc/win)
+    #   (media_test/all_tests)
+    #   (small/med/large)
+    # How to handle that?
+
+    # Return classification if the file is included and has a group and
+    # language
+    if include and group and language:
+      return group, language
+    else:
+      return None, None
+
+  def AddRoot(self, root_path, alt_name='#'):
+    """Adds a root directory.
+
+    Args:
+      root_path: Root directory to add.
+      alt_name: If specified, name of root dir
+    """
+    # Clean up root path based on existing rules
+    self.root_dirs.append([self.CleanupFilename(root_path), alt_name])
+
+  def AddRule(self, path_regexp, include=None, group=None, language=None):
+    """Adds a rule.
+
+    Args:
+      path_regexp: Regular expression to match for filenames.  These are
+          matched after root directory replacement.
+      include: If True, includes matches; if False, excludes matches.  Ignored
+          if None.
+      group: If not None, sets group to apply to matches.
+      language: If not None, sets file language to apply to matches.
+    """
+    # Compile regexp ahead of time
+    self.rules.append([re.compile(path_regexp), include, group, language])
+
+  def GetCoveredFile(self, filename, add=False):
+    """Gets the CoveredFile object for the filename.
+
+    Args:
+      filename: Name of file to find.
+      add: If True, will add the file if it's not present.  This applies the
+          transformations from AddRoot() and AddRule(), and only adds the file
+          if a rule includes it, and it has a group and language.
+
+    Returns:
+      The matching CoveredFile object, or None if not present.
+    """
+    # Clean filename
+    filename = self.CleanupFilename(filename)
+
+    # Check for existing match
+    if filename in self.files:
+      return self.files[filename]
+
+    # File isn't one we know about.  If we can't add it, give up.
+    if not add:
+      return None
+
+    # Check rules to see if file can be added
+    group, language = self.ClassifyFile(filename)
+    if not group:
+      return None
+
+    # Add the file
+    f = CoveredFile(filename, group, language)
+    self.files[filename] = f
+
+    # Return the newly covered file
+    return f
+
+  def ParseLcovData(self, lcov_data):
+    """Adds coverage from LCOV-formatted data.
+
+    Args:
+      lcov_data: An iterable returning lines of data in LCOV format.  For
+          example, a file or list of strings.
+    """
+    cov_file = None
+    cov_lines = None
+    for line in lcov_data:
+      line = line.strip()
+      if line.startswith('SF:'):
+        # Start of data for a new file; payload is filename
+        cov_file = self.GetCoveredFile(line[3:], add=True)
+        if cov_file:
+          cov_lines = cov_file.lines
+      elif not cov_file:
+        # Inside data for a file we don't care about - so skip it
+        pass
+      elif line.startswith('DA:'):
+        # Data point - that is, an executable line in current file
+        line_no, is_covered = map(int, line[3:].split(','))
+        if is_covered:
+          # Line is covered
+          cov_lines[line_no] = 1
+        elif cov_lines.get(line_no) != 1:
+          # Line is not covered, so track it as uncovered
+          cov_lines[line_no] = 0
+      elif line == 'end_of_record':
+        cov_file.UpdateCoverage()
+        cov_file = None
+      # (else ignore other line types)
+
+  def ParseLcovFile(self, input_filename):
+    """Adds coverage data from a .lcov file.
+
+    Args:
+      input_filename: Input filename.
+    """
+    # TODO: All manner of error checking
+    lcov_file = None
+    try:
+      lcov_file = open(input_filename, 'rt')
+      self.ParseLcovData(lcov_file)
+    finally:
+      if lcov_file:
+        lcov_file.close()
+
+  def GetStat(self, stat, group='all', default=None):
+    """Gets a statistic from the coverage object.
+
+    Args:
+      stat: Statistic to get.  May also be an evaluatable python expression,
+          using the stats.  For example, 'stat1 - stat2'.
+      group: File group to match; if 'all', matches all groups.
+      default: Value to return if there was an error evaluating the stat.  For
+          example, if the stat does not exist.  If None, raises
+          CoverageStatError.
+
+    Returns:
+      The evaluated stat, or None if error.
+
+    Raises:
+      CoverageStatError: Error evaluating stat.
+    """
+    # TODO: specify a subdir to get the stat from, then walk the tree to
+    # print the stats from just that subdir
+
+    # Make sure the group exists
+    if group not in self.tree.stats_by_group:
+      if default is None:
+        raise CoverageStatError('Group %r not found.' % group)
+      else:
+        return default
+
+    stats = self.tree.stats_by_group[group]
+    try:
+      return eval(stat, {'__builtins__':{'S':self.GetStat}}, stats)
+    except Exception, e:
+      if default is None:
+        raise CoverageStatError('Error evaluating stat %r: %s' % (stat, e))
+      else:
+        return default
+
+  def PrintStat(self, stat, format=None, outfile=sys.stdout, **kwargs):
+    """Prints a statistic from the coverage object.
+
+    Args:
+      stat: Statistic to get.  May also be an evaluatable python expression,
+          using the stats.  For example, 'stat1 - stat2'.
+      format: Format string to use when printing stat.  If None, prints the
+          stat and its evaluation.
+      outfile: File stream to output stat to; defaults to stdout.
+      kwargs: Additional args to pass to GetStat().
+    """
+    s = self.GetStat(stat, **kwargs)
+    if format is None:
+      outfile.write('GetStat(%r) = %s\n' % (stat, s))
+    else:
+      outfile.write(format % s + '\n')
+
+  def AddFiles(self, src_dir):
+    """Adds files to coverage information.
+
+    LCOV files only contains files which are compiled and instrumented as part
+    of running coverage.  This function finds missing files and adds them.
+
+    Args:
+      src_dir: Directory on disk at which to start search.  May be a relative
+          path on disk starting with '.' or '..', or an absolute path, or a
+          path relative to an alt_name for one of the roots
+          (for example, '#/src').  If the alt_name matches more than one root,
+          all matches will be attempted.
+
+    Note that dirs not underneath one of the root dirs and covered by an
+    inclusion rule will be ignored.
+    """
+    # Check for root dir alt_names in the path and replace with the actual
+    # root dirs, then recurse.
+    found_root = False
+    for root, alt_name in self.root_dirs:
+      replaced_root = re.sub('^' + re.escape(alt_name) + '(?=(/|$))', root,
+                             src_dir)
+      if replaced_root != src_dir:
+        found_root = True
+        self.AddFiles(replaced_root)
+    if found_root:
+      return    # Replaced an alt_name with a root_dir, so already recursed.
+
+    for (dirpath, dirnames, filenames) in self.add_files_walk(src_dir):
+      # Make a copy of the dirnames list so we can modify the original to
+      # prune subdirs we don't need to walk.
+      for d in list(dirnames):
+        # Add trailing '/' to directory names so dir-based regexps can match
+        # '/' instead of needing to specify '(/|$)'.
+        dpath = self.CleanupFilename(dirpath + '/' + d) + '/'
+        group, language = self.ClassifyFile(dpath)
+        if not group:
+          # Directory has been excluded, so don't traverse it
+          # TODO: Document the slight weirdness caused by this: If you
+          # AddFiles('./A'), and the rules include 'A/B/C/D' but not 'A/B',
+          # then it won't recurse into './A/B' so won't find './A/B/C/D'.
+          # Workarounds are to AddFiles('./A/B/C/D') or AddFiles('./A/B/C').
+          # The latter works because it explicitly walks the contents of the
+          # path passed to AddFiles(), so it finds './A/B/C/D'.
+          dirnames.remove(d)
+
+      for f in filenames:
+        covf = self.GetCoveredFile(dirpath + '/' + f, add=True)
+        # TODO: scan files for executable lines.  Add these to the file as
+        # 'executable', but not 'instrumented' or 'covered'.
+        # TODO: if a file has no executable lines, don't add it.
+        if covf:
+          covf.UpdateCoverage()
+
+  def AddConfig(self, config_data, lcov_queue=None, addfiles_queue=None):
+    """Adds JSON-ish config data.
+
+    Args:
+      config_data: Config data string.
+      lcov_queue: If not None, object to append lcov_files to instead of
+          parsing them immediately.
+      addfiles_queue: If not None, object to append add_files to instead of
+          processing them immediately.
+    """
+    # TODO: All manner of error checking
+    cfg = eval(config_data, {'__builtins__':{}}, {})
+
+    for rootdict in cfg.get('roots', []):
+      self.AddRoot(rootdict['root'], alt_name=rootdict.get('altname', '#'))
+
+    for ruledict in cfg.get('rules', []):
+      self.AddRule(ruledict['regexp'],
+                   include=ruledict.get('include'),
+                   group=ruledict.get('group'),
+                   language=ruledict.get('language'))
+
+    for add_lcov in cfg.get('lcov_files', []):
+      if lcov_queue is not None:
+        lcov_queue.append(add_lcov)
+      else:
+        self.ParseLcovFile(add_lcov)
+
+    for add_path in cfg.get('add_files', []):
+      if addfiles_queue is not None:
+        addfiles_queue.append(add_path)
+      else:
+        self.AddFiles(add_path)
+
+    self.print_stats += cfg.get('print_stats', [])
+
+  def ParseConfig(self, filename, **kwargs):
+    """Parses a configuration file.
+
+    Args:
+      filename: Config filename.
+    """
+    # TODO: All manner of error checking
+    f = None
+    try:
+      f = open(filename, 'rt')
+      # Need to strip CR's from CRLF-terminated lines or posix systems can't
+      # eval the data.
+      config_data = f.read().replace('\r\n', '\n')
+      # TODO: some sort of include syntax.  Needs to be done at string-time
+      # rather than at eval()-time, so that it's possible to include parts of
+      # dicts.  Path from a file to its include should be relative to the dir
+      # containing the file.
+      self.AddConfig(config_data, **kwargs)
+    finally:
+      if f:
+        f.close()
+
+  def UpdateTreeStats(self):
+    """Recalculates the tree stats from the currently covered files.
+
+    Also calculates coverage summary for files."""
+    self.tree = CoveredDir('')
+    for cov_file in self.files.itervalues():
+      # Add the file to the tree
+      # TODO: Don't really need to create the tree unless we're creating HTML
+      fdirs = cov_file.filename.split('/')
+      parent = self.tree
+      ancestors = [parent]
+      for d in fdirs[:-1]:
+        if d not in parent.subdirs:
+          parent.subdirs[d] = CoveredDir(d)
+        parent = parent.subdirs[d]
+        ancestors.append(parent)
+      # Final subdir actually contains the file
+      parent.files[fdirs[-1]] = cov_file
+
+      # Now add file's contribution to coverage by dir
+      for a in ancestors:
+        # Add to 'all' group
+        a.stats_by_group['all'].Add(cov_file.stats)
+
+        # Add to group file belongs to
+        if cov_file.group not in a.stats_by_group:
+          a.stats_by_group[cov_file.group] = CoverageStats()
+        cbyg = a.stats_by_group[cov_file.group]
+        cbyg.Add(cov_file.stats)
+
+  def PrintTree(self):
+    """Prints the tree stats."""
+    # Print the tree
+    print 'Lines of code coverage by directory:'
+    print self.tree.GetTree()
+
+#------------------------------------------------------------------------------
+
+
+def Main(argv):
+  """Main routine.
+
+  Args:
+    argv: list of arguments
+
+  Returns:
+    exit code, 0 for normal exit.
+  """
+  # Parse args
+  parser = OptionParser()
+  parser.add_option(
+      '-i', '--input', dest='inputs', type='string', action='append',
+      metavar='FILE',
+      help='read LCOV input from FILE')
+  parser.add_option(
+      '-r', '--root', dest='roots', type='string', action='append',
+      metavar='ROOT[=ALTNAME]',
+      help='add ROOT directory, optionally map in coverage results as ALTNAME')
+  parser.add_option(
+      '-c', '--config', dest='configs', type='string', action='append',
+      metavar='FILE',
+      help='read settings from configuration FILE')
+  parser.add_option(
+      '-a', '--addfiles', dest='addfiles', type='string', action='append',
+      metavar='PATH',
+      help='add files from PATH to coverage data')
+  parser.add_option(
+      '-t', '--tree', dest='tree', action='store_true',
+      help='print tree of code coverage by group')
+  parser.add_option(
+      '-u', '--uninstrumented', dest='uninstrumented', action='store_true',
+      help='list uninstrumented files')
+
+  parser.set_defaults(
+      inputs=[],
+      roots=[],
+      configs=[],
+      addfiles=[],
+      tree=False,
+  )
+
+  (options, args) = parser.parse_args()
+
+  cov = Coverage()
+
+  # Set root directories for coverage
+  for root_opt in options.roots:
+    if '=' in root_opt:
+      cov.AddRoot(*root_opt.split('='))
+    else:
+      cov.AddRoot(root_opt)
+
+  # Read config files
+  for config_file in options.configs:
+    cov.ParseConfig(config_file, lcov_queue=options.inputs,
+                    addfiles_queue=options.addfiles)
+
+  # Parse lcov files
+  for input_filename in options.inputs:
+    cov.ParseLcovFile(input_filename)
+
+  # Add missing files
+  for add_path in options.addfiles:
+    cov.AddFiles(add_path)
+
+  # Print help if no files specified
+  if not cov.files:
+    print 'No covered files found.'
+    parser.print_help()
+    return 1
+
+  # Update tree stats
+  cov.UpdateTreeStats()
+
+  # Print uninstrumented filenames
+  if options.uninstrumented:
+    print 'Uninstrumented files:'
+    for f in sorted(cov.files):
+      covf = cov.files[f]
+      if not covf.stats.get('lines_instrumented'):
+        print '  %-6s %-6s %s' % (covf.group, covf.language, f)
+
+
+  # Print tree stats
+  if options.tree:
+    cov.PrintTree()
+
+  # Print stats
+  for ps_args in cov.print_stats:
+    cov.PrintStat(**ps_args)
+
+  # Normal exit
+  return 0
+
+
+#------------------------------------------------------------------------------
+
+if __name__ == '__main__':
+  sys.exit(Main(sys.argv))