Changes to code coverage infrastructure to allow subset coverage analysis.

tools/code_coverage/coverage_posix.py

 #!/usr/bin/env python
 # Copyright (c) 2011 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Generate and process code coverage.
 
 TODO(jrg): rename this from coverage_posix.py to coverage_all.py!
 
 Written for and tested on Mac, Linux, and Windows.  To use this script
@@ skipping 76 matching lines @@
 have all text before a ':' stripped to help with gyp compatibility.
 For example, ../base/base.gyp:base_unittests is interpreted as a test
 named "base_unittests".
 """
 
 import glob
 import logging
 import optparse
 import os
 import Queue
+import re
 import shutil
 import signal
 import subprocess
 import sys
 import tempfile
 import threading
 import time
 import traceback
 
 
@@ skipping 32 matching lines @@
       os.kill(pid, sig)
     else:
       subprocess.call(['taskkill.exe', '/PID', str(pid)])
   sys.exit(0)
 
 
 class RunTooLongException(Exception):
   """Thrown when a command runs too long without output."""
   pass
 
+class BadUserInput(Exception):
+  """Thrown when arguments from the user are incorrectly formatted."""
+  pass
+
 
 class RunProgramThread(threading.Thread):
   """A thread to run a subprocess.
 
   We want to print the output of our subprocess in real time, but also
   want a timeout if there has been no output for a certain amount of
   time.  Normal techniques (e.g. loop in select()) aren't cross
   platform enough. the function seems simple: "print output of child, kill it
   if there is no output by timeout.  But it was tricky to get this right
   in a x-platform way (see warnings about deadlock on the python
@@ skipping 150 matching lines @@
     # The "final" lcov-format file
     self.coverage_info_file = os.path.join(self.directory, 'coverage.info')
     # If needed, an intermediate VSTS-format file
     self.vsts_output = os.path.join(self.directory, 'coverage.vsts')
     # Needed for Windows.
     self.src_root = options.src_root
     self.FindPrograms()
     self.ConfirmPlatformAndPaths()
     self.tests = []
     self.xvfb_pid = 0
+    self.test_files = []        # List of files with test specifications.
+    self.test_filters = {}      # Mapping from testname->--gtest_filter arg.
     logging.info('self.directory: ' + self.directory)
     logging.info('self.directory_parent: ' + self.directory_parent)
 
   def FindInPath(self, program):
     """Find program in our path.  Return abs path to it, or None."""
     if not 'PATH' in os.environ:
       logging.fatal('No PATH environment variable?')
       sys.exit(1)
     paths = os.environ['PATH'].split(os.pathsep)
     for path in paths:
@@ skipping 101 matching lines @@
     Assume all non-option items in the arg list are tests to be run.
     """
     # Before we begin, find the bundles file if not an absolute path.
     self.FindBundlesFile()
 
     # Small tests: can be run in the "chromium" directory.
     # If asked, run all we can find.
     if self.options.all_unittests:
       self.tests += glob.glob(os.path.join(self.directory, '*_unittests'))
 
-    # Tests can come in as args or as a file of bundles.
+    # Tests can come in as args directly, indirectly (through a file
+    # of test lists) or as a file of bundles.
     all_testnames = self.args[:]  # Copy since we might modify
+
+    for test_file in self.options.test_files:
+      f = open(test_file)
+      for line in f:
+        line = re.sub(r"#.*$", "", line)
+        line = re.sub(r"\s*", "", line)
+        if re.match("\s*$"):
+          continue
+        all_testnames.append(line)
+      f.close()
+
     tests_from_bundles = None
     if self.options.bundles:
       try:
         tests_from_bundles = eval(open(self.options.bundles).read())
       except IOError:
         logging.fatal('IO error in bundle file ' +
                       self.options.bundles + ' (doesn\'t exist?)')
       except (NameError, SyntaxError):
         logging.fatal('Parse or syntax error in bundle file ' +
                       self.options.bundles)
       if hasattr(tests_from_bundles, '__iter__'):
         all_testnames += tests_from_bundles
       else:
         logging.fatal('Fatal error with bundle file; could not get list from' +
                       self.options.bundles)
         sys.exit(1)
 
     # If told explicit tests, run those (after stripping the name as
     # appropriate)
     for testname in all_testnames:
+      mo = re.search(r"(.*)\[(.*)\]$", testname)
+      gtest_filter = None
+      if mo:
+        gtest_filter = mo.group(2)
+        testname = mo.group(1)
+
       if ':' in testname:
-        self.tests += [os.path.join(self.directory, testname.split(':')[1])]
-      else:
-        self.tests += [os.path.join(self.directory, testname)]
+        testname = testname.split(':')[1]
+      self.tests += [os.path.join(self.directory, testname)]
+      if gtest_filter:
+        self.test_filters[testname] = gtest_filter
+
     # Medium tests?
     # Not sure all of these work yet (e.g. page_cycler_tests)
     # self.tests += glob.glob(os.path.join(self.directory, '*_tests'))
 
     # If needed, append .exe to tests since vsinstr.exe likes it that
     # way.
     if self.IsWindows():
       for ind in range(len(self.tests)):
         test = self.tests[ind]
         test_exe = test + '.exe'
@@ skipping 117 matching lines @@
   def GtestFilter(self, fulltest, excl=None):
     """Return a --gtest_filter=BLAH for this test.
 
     Args:
       fulltest: full name of test executable
       exclusions: the exclusions list.  Only set in a unit test;
         else uses gTestExclusions.
     Returns:
       String of the form '--gtest_filter=BLAH', or None.
     """
-    if self.options.no_exclusions:
-      return
-    exclusions = excl or gTestExclusions
-    excldict = exclusions.get(sys.platform)
-    if excldict:
-      for test in excldict.keys():
-        # example: if base_unittests in ../blah/blah/base_unittests.exe
-        if test in fulltest:
-          gfilter = '--gtest_filter=-' + ':-'.join(excldict[test])
-          return gfilter
-    return None
+    positive_gfilter_list = []
+    negative_gfilter_list = []
+
+    # Exclude all flaky and failing tests; they don't count for code coverage.
+    negative_gfilter_list += ("*.FLAKY_*", "*.FAILS_*")
+
+    if not self.options.no_exclusions:
+      exclusions = excl or gTestExclusions
+      excldict = exclusions.get(sys.platform)
+      if excldict:
+        for test in excldict.keys():
+          # example: if base_unittests in ../blah/blah/base_unittests.exe
+          if test in fulltest:
+            negative_gfilter_list += excldict[test]
+
+    fulltest_basename = os.path.basename(fulltest)
+    if fulltest_basename in self.test_filters:
+      specific_test_filters = self.test_filters[fulltest_basename].split("-")
+      if len(specific_test_filters) > 2:
+        logging.error("Multiple '-' symbols in filter list: %s" %
+          self.test_filters[fulltest_basename])
+        raise BadUserInput()
+      if len(specific_test_filters) == 2:
+        # Remove trailing ":"
+        specific_test_filters[0] = specific_test_filters[0][:-1]
+
+      if specific_test_filters[0]: # Test for no positive filters.
+        positive_gfilter_list += specific_test_filters[0].split(":")
+      if len(specific_test_filters) > 1:
+        negative_gfilter_list += specific_test_filters[1].split(":")
+
+    if not positive_gfilter_list and not negative_gfilter_list:
+      return None
+
+    result = "--gtest_filter="
+    if positive_gfilter_list:
+      result += ":".join(positive_gfilter_list)
+    if negative_gfilter_list:
+      if positive_gfilter_list: result += ":"
+      result += "-" + ":".join(negative_gfilter_list)
+    return result
 
   def RunTests(self):
     """Run all unit tests and generate appropriate lcov files."""
     self.BeforeRunAllTests()
     for fulltest in self.tests:
       if not os.path.exists(fulltest):
         logging.info(fulltest + ' does not exist')
         if self.options.strict:
           sys.exit(2)
       else:
@@ skipping 257 matching lines @@
                           'used for finding bundlefile (e.g. Debug)'))
   parser.add_option('--no_exclusions',
                     dest='no_exclusions',
                     default=None,
                     help=('Disable the exclusion list.'))
   parser.add_option('--dont-clear-coverage-data',
                     dest='dont_clear_coverage_data',
                     default=False,
                     action='store_true',
                     help=('Turn off clearing of cov data from a prev run'))
+  parser.add_option('-F',
+                    '--test-file',
+                    dest="test_files",
+                    default=[],
+                    action="append",
+                    help=('Specify a file from which tests to be run will ' +
+                          'be extracted'))
   return parser
 
 
 def main():
   # Print out the args to help someone do it by hand if needed
   print >>sys.stderr, sys.argv
 
   # Try and clean up nice if we're killed by buildbot, Ctrl-C, ...
   signal.signal(signal.SIGINT, TerminateSignalHandler)
   signal.signal(signal.SIGTERM, TerminateSignalHandler)
 
   parser = CoverageOptionParser()
   (options, args) = parser.parse_args()
   coverage = Coverage(options, args)
   coverage.ClearData()
   coverage.FindTests()
   if options.trim:
     coverage.TrimTests()
   coverage.RunTests()
   if options.genhtml:
     coverage.GenerateHtml()
   return 0
 
 
 if __name__ == '__main__':
   sys.exit(main())