Config files

expect_tests/pipeline.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-import Queue
+import ConfigParser
 import glob
 import imp
 import inspect
 import logging
 import multiprocessing
 import os
+import Queue
 import re
 import signal
 import sys
 import tempfile
 import traceback
 
 from cStringIO import StringIO
 
 from expect_tests.tempdir import TempDir
 from expect_tests.type_definitions import (
     Test, UnknownError, TestError, NoMatchingTestsError, MultiTest,
     Result, ResultStageAbort)
 
 from expect_tests.unittest_helper import _is_unittest, UnittestTestCase
 from expect_tests import util
 
 
+CONFIG_FILE_NAME='.expect_tests.cfg'
+
+
 class ResetableStringIO(object):
   def __init__(self):
     self._stream = StringIO()
 
   def reset(self):
     self._stream = StringIO()
 
   def __getattr__(self, key):
     return getattr(self._stream, key)
 
 
 def get_package_path(package_name, path):
   """Return path toward 'package_name'.
 
   If path is None, search for a package in sys.path.
   Otherwise, look for a direct subdirectory of path.
 
   If no package is found, returns None.
   """
   if path is None:
     _, package_path, _ = imp.find_module(package_name)
   else:
     package_path = os.path.join(path, package_name)
     if not os.path.exists(package_path):
       raise ValueError('File not found: %s' % package_path)
   ispkg = os.path.isfile(os.path.join(package_path, '__init__.py'))
   return package_path if ispkg else None
 
 
-def get_blacklist(path):
+def get_config(path):
   """Get blacklisted subdirectories
 
-  Reads the file called __expect_tests_skip in provided path, remove
-  comments, and returns content.
+  Reads the config file in provided path, and returns content.
 
   A comment starts with #, on its own line (no trailing comment).
 
   Args:
   path (str): path to a directory.
 
   Returns:
   blacklist (set of str): name of blacklisted subdirectories.
   """
   black_list = set()
-  black_list_filename = os.path.join(path, '__expect_tests_skip')
-  if os.path.isfile(black_list_filename):
-    with open(black_list_filename, 'r') as f:
-      for line in f:
-        stripped = line.strip()
-        if stripped and not stripped.startswith('#'):
-          black_list.add(stripped)
+  python_path = ''
+  black_list_filename = os.path.join(path, CONFIG_FILE_NAME)

[dnj, 2014/09/19 21:15:59]

If you're generalizing this to config file, the variable sholud be
"config_filename" or thereabouts.

[pgervais, 2014/09/19 22:06:26]

Done.

+  parser = ConfigParser.ConfigParser()
+  parser.read([black_list_filename])
 
-  return black_list
+  if not parser.has_section('expect_tests'):
+    return black_list, python_path
+
+  if parser.has_option('expect_tests', 'skip'):
+    black_list = parser.get('expect_tests', 'skip').splitlines()
+  if parser.has_option('expect_tests', 'pythonpath'):
+    python_path = [os.path.normpath(os.path.join(path, s.strip()))
+                   for s
+                   in parser.get('expect_tests', 'pythonpath').splitlines()]
+  return black_list, python_path
 
 
-def walk_package(package_name, path):
-  """Yield all test files inside a single package.
+def walk_package(package_name, path=None):
+  """Return all test files inside a single package.
 
   In all cases, this function returns the full package name of files ending
   in '_test.py' found either under the package called <package_name>.
   Example: 'test_package.foo_test' for 'test_package/foo_test.py'.
 
   Provided that <path> is in sys.path, calling __import__ with one of the
   strings returned by this function works.
 
-  If a __expect_tests_skip file is present somewhere in the search hierarchy,
+  If a config file is present somewhere in the search hierarchy,
   it is interpreted as a list of subdirectories to ignore. This is the way to
   make this function ignore some subpackages.
 
   This function has several behaviors depending on the arguments:
   - if path is None: search for package_name under any path in sys.path.
   - if path is not None: search for a package called package_name directly
     under <path> (sys.path is not used).
 
   It is not necessary to change sys.path for the present function to work (it
   does not actually import anything).
 
+  Args:
+  package_name (str): name of the package, as expected by import.
+  path (str): path containing the above module (optional)
+
+  Returns:
+  test_modules (list of str): name of modules containing tests. Each element is
+     a period-separated string ending with '_test',
+     e.g. shiny_package.subpackage.feature_test
+  pythonpath (list of str): paths to add to sys.path, as requested in
+     the config files.
+
   Example:
-  modules = walk_package('shiny_package', 'some/directory')
-  sys.path.insert(0, 'some/directory')
+  modules, pythonpath = walk_package('shiny_package', 'some/directory')
+  sys.path[:] = ['some/directory'] + pythonpath + sys.path
   __import__(modules[0])
 
   the first line assumes that the directory 'some/directory' is in the
     current working directory.
-
-  modules[0] is a period-separated string ending with '_test',
-  e.g. shiny_package.subpackage.feature_test
   """
   assert package_name, 'walk_package needs a package_name.'
 
   # This function can be called with submodule names, but it's not a public
   # interface.
   subpackage_name = package_name.split('.')[-1]
 
-  ret = []
+  pythonpath = []
+  test_modules = []
   package_path = get_package_path(subpackage_name, path)
   assert package_path, 'no package found.'
 
   # Look for direct subpackages and test files.
-  black_list = get_blacklist(package_path)
+  black_list, extra_paths = get_config(package_path)
+  pythonpath.extend(extra_paths)
   for filename in filter(lambda x: x not in black_list,
                          os.listdir(package_path)):
     if os.path.isfile(os.path.join(package_path, filename, '__init__.py')):
-      ret += walk_package(
+      extra_test_modules, extra_paths = walk_package(
         (package_name + '.' if package_name else '')
         + filename, package_path)
+      test_modules.extend(extra_test_modules)
+      pythonpath.extend(extra_paths)
       continue
     if filename.endswith('_test.py'):
-      ret.append(
+      test_modules.append(
         (package_name + '.' if package_name else '')
         + inspect.getmodulename(filename))
-  return ret
+  return test_modules, pythonpath
 
 
 def load_module(modname):
   """Import and return the specified module.
 
   Uses __import__ instead of pkgutil's PEP302-style find_module/load_module
   because those just don't work. The tradeoff is that we have to walk down the
   package hierarchy to find the leaf module (since __import__ returns the
   topmost parent package), but at least this behaves deterministically.
   """
@@ skipping 10 matching lines @@
   See UnittestTestCase for possible return values.
 
   This function loads modules, thus no two conflicting packages (like appengine)
   can be loaded at the same time: use separate processes for that.
   """
   assert isinstance(path, basestring), 'path must be a string'
   assert os.path.isdir(path), 'path is not a directory: %s' % path
   sys.path.insert(0, os.path.abspath(path))
 
   test_gens = []
-  black_list = get_blacklist(path)
+  black_list, _ = get_config(path)
 
   for filename in filter(lambda x: x not in black_list, os.listdir(path)):
     abs_filename = os.path.join(path, filename)
     if (os.path.isdir(abs_filename)
         and os.path.isfile(os.path.join(abs_filename, '__init__.py'))):
       test_gens += get_test_gens_package(abs_filename,
                                          update_syspath=False)
   return test_gens
 
 
@@ skipping 14 matching lines @@
   assert os.path.isfile(os.path.join(package, '__init__.py')), \
          "'package' is not pointing to a package. It must be a " + \
          "path to a directory containing a __init__.py file."
 
   test_gens = []
   path = os.path.abspath(os.path.dirname(package))
   if update_syspath:
     sys.path.insert(0, path)
   package_name = os.path.split(package.rstrip(os.path.sep))[-1]
 
-  for modname in walk_package(package_name, path):
+  test_modules, pythonpath = walk_package(package_name, path)
+
+  if pythonpath:
+    sys.path[:] = pythonpath + sys.path
+
+  for modname in test_modules:
     mod = load_module(modname)
     for obj in mod.__dict__.values():
       if util.is_test_generator(obj):
         test_gens.append(obj)
       elif _is_unittest(obj):
         test_gens.append(UnittestTestCase(obj))
   return test_gens
 
 
 def gen_loop_process(gens, test_queue, result_queue, opts, kill_switch,
@@ skipping 293 matching lines @@
 
     if procs:
       error = opts.handler.result_stage_loop(opts, generate_objects(procs))
   except ResultStageAbort:
     pass
 
   if not kill_switch.is_set() and not result_queue.empty():
     error = True
 
   return error, kill_switch.is_set()