Config files

expect_tests/pipeline.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-import Queue
+import ConfigParser
 import glob
 import imp
 import inspect
 import logging
 import multiprocessing
 import os
+import Queue
 import re
 import signal
 import sys
 import tempfile
 import traceback
 
 from cStringIO import StringIO
 
 from expect_tests.tempdir import TempDir
 from expect_tests.type_definitions import (
     Test, UnknownError, TestError, NoMatchingTestsError, MultiTest,
     Result, ResultStageAbort)
 
 from expect_tests.unittest_helper import _is_unittest, UnittestTestCase
 from expect_tests import util
 
 
+CONFIG_FILE_NAME = '.expect_tests.cfg'
+
+
 class ResetableStringIO(object):
   def __init__(self):
     self._stream = StringIO()
 
   def reset(self):
     self._stream = StringIO()
 
   def __getattr__(self, key):
     return getattr(self._stream, key)
 
 
 def get_package_path(package_name, path):
   """Return path toward 'package_name'.
 
   If path is None, search for a package in sys.path.
   Otherwise, look for a direct subdirectory of path.
 
   If no package is found, returns None.
   """
   if path is None:
     _, package_path, _ = imp.find_module(package_name)
   else:
     package_path = os.path.join(path, package_name)
     if not os.path.exists(package_path):
       raise ValueError('File not found: %s' % package_path)
   ispkg = os.path.isfile(os.path.join(package_path, '__init__.py'))
   return os.path.normpath(package_path) if ispkg else None
 
 
-def get_blacklist(path):
-  """Get blacklisted subdirectories
+def get_config(path):
+  """Get configuration values
 
-  Reads the file called __expect_tests_skip in provided path, remove
-  comments, and returns content.
+  Reads the config file in provided path, and returns content.
+  See Python ConfigParser for general formatting syntax.
 
-  A comment starts with #, on its own line (no trailing comment).
+  Example:
+  [expect_tests]
+  skip=directory1
+     directory2
+     directory3
 
   Args:
   path (str): path to a directory.
 
   Returns:
-  blacklist (set of str): name of blacklisted subdirectories.
+  black_list (set): blacklisted subdirectories.
   """
   black_list = set()
-  black_list_filename = os.path.join(path, '__expect_tests_skip')
-  if os.path.isfile(black_list_filename):
-    with open(black_list_filename, 'r') as f:
-      for line in f:
-        stripped = line.strip()
-        if stripped and not stripped.startswith('#'):
-          black_list.add(stripped)
 
+  config_file_name = os.path.join(path, CONFIG_FILE_NAME)
+  parser = ConfigParser.ConfigParser()
+  parser.read([config_file_name])
+
+  if not parser.has_section('expect_tests'):
+    return black_list
+
+  if parser.has_option('expect_tests', 'skip'):
+    black_list = parser.get('expect_tests', 'skip').splitlines()

[dnj, 2014/09/19 23:56:05]

This should be 'black_list.update(...)'. Function says it returns a set, plus
otherwise you're creating and then discarding the default empty set.

   return black_list
 
 
 def is_test_file(filename):
   """Returns True if filename is supposed to contain tests.
 
   Args:
   filename (str): path to a python file.
 
   Returns:
   is_test (boolean): True if filename points to a test file.
   """
   return filename.endswith('_test.py')
 
 
 def walk_package(package_name, path):
-  """Yield all test files inside a single package.
+  """Return all test files inside a single package.
 
   In all cases, this function returns the full package name of files ending
   in '_test.py' found either under the package called <package_name>.
   Example: 'test_package.foo_test' for 'test_package/foo_test.py'.
 
   Provided that <path> is in sys.path, calling __import__ with one of the
   strings returned by this function works.
 
-  If a __expect_tests_skip file is present somewhere in the search hierarchy,
+  If a config file is present somewhere in the search hierarchy,
   it is interpreted as a list of subdirectories to ignore. This is the way to
   make this function ignore some subpackages.
 
   This function has several behaviors depending on the arguments:
   - if path is None: search for package_name under any path in sys.path.
   - if path is not None: search for a package called package_name directly
     under <path> (sys.path is not used).
 
   It is not necessary to change sys.path for the present function to work (it
   does not actually import anything).
 
+  Args:
+  package_name (str): name of the package, as expected by import.
+  path (str): path containing the above module (optional)
+
+  Returns:
+  test_modules (list of str): name of modules containing tests. Each element is
+     a period-separated string ending with '_test',
+     e.g. shiny_package.subpackage.feature_test
+
   Example:
   modules = walk_package('shiny_package', 'some/directory')
-  sys.path.insert(0, 'some/directory')
+  sys.path[:] = sys.path.insert(0, 'some/directory')
   __import__(modules[0])
 
   the first line assumes that the directory 'some/directory' is in the
     current working directory.
-
-  modules[0] is a period-separated string ending with '_test',
-  e.g. shiny_package.subpackage.feature_test
   """
   assert package_name, 'walk_package needs a package_name.'
 
-  ret = []
+  test_modules = []
   package_path = get_package_path(package_name, path)
   assert package_path, 'no package found.'
 
   base_path = os.path.split(package_path.rstrip(os.path.sep))[0]
   assert package_path.startswith(base_path)
 
   for dirpath, dirnames, filenames in os.walk(package_path):
     # Keep only submodules not blacklisted
-    blacklist = get_blacklist(dirpath)
-
+    blacklist = get_config(dirpath)
     dirnames[:] = [d for d in dirnames
                    if d not in blacklist and
                    os.path.isfile(os.path.join(dirpath, d, '__init__.py'))]
 
     assert dirpath.startswith(package_path)
     base_module_name = os.path.relpath(dirpath, base_path).split(os.path.sep)
-    ret += ['.'.join(base_module_name + [inspect.getmodulename(filename)])
-            for filename in filenames
-            if is_test_file(filename)]
+    test_modules += ['.'.join(base_module_name
+                              + [inspect.getmodulename(filename)])
+                     for filename in filenames
+                     if is_test_file(filename)]
 
-  return ret
+  return test_modules
 
 
 def load_module(modname):
   """Import and return the specified module.
 
   Uses __import__ instead of pkgutil's PEP302-style find_module/load_module
   because those just don't work. The tradeoff is that we have to walk down the
   package hierarchy to find the leaf module (since __import__ returns the
   topmost parent package), but at least this behaves deterministically.
   """
@@ skipping 10 matching lines @@
   See UnittestTestCase for possible return values.
 
   This function loads modules, thus no two conflicting packages (like appengine)
   can be loaded at the same time: use separate processes for that.
   """
   assert isinstance(path, basestring), 'path must be a string'
   assert os.path.isdir(path), 'path is not a directory: %s' % path
   sys.path.insert(0, os.path.abspath(path))
 
   test_gens = []
-  black_list = get_blacklist(path)
+  black_list = get_config(path)
 
   for filename in filter(lambda x: x not in black_list, os.listdir(path)):
     abs_filename = os.path.join(path, filename)
     if (os.path.isdir(abs_filename)
         and os.path.isfile(os.path.join(abs_filename, '__init__.py'))):
       test_gens += get_test_gens_package(abs_filename,
                                          update_syspath=False)
   return test_gens
 
 
@@ skipping 14 matching lines @@
   assert os.path.isfile(os.path.join(package, '__init__.py')), \
          "'package' is not pointing to a package. It must be a " + \
          "path to a directory containing a __init__.py file."
 
   test_gens = []
   path = os.path.abspath(os.path.dirname(package))
   if update_syspath:
     sys.path.insert(0, path)
   package_name = os.path.split(package.rstrip(os.path.sep))[-1]
 
-  for modname in walk_package(package_name, path):
+  test_modules = walk_package(package_name, path)
+
+  for modname in test_modules:

[dnj, 2014/09/19 23:56:05]

nit: Why split this? I think it was fine before.

[pgervais, 2014/09/20 00:26:55]

merge problem I think. will fix.

     mod = load_module(modname)
     for obj in mod.__dict__.values():
       if util.is_test_generator(obj):
         test_gens.append(obj)
       elif _is_unittest(obj):
         test_gens.append(UnittestTestCase(obj))
   return test_gens
 
 
 def gen_loop_process(gens, test_queue, result_queue, opts, kill_switch,
@@ skipping 292 matching lines @@
 
     if procs:
       error = opts.handler.result_stage_loop(opts, generate_objects(procs))
   except ResultStageAbort:
     pass
 
   if not kill_switch.is_set() and not result_queue.empty():
     error = True
 
   return error, kill_switch.is_set()