New expect_tests UI

expect_tests/pipeline.py

Index: expect_tests/pipeline.py
diff --git a/expect_tests/pipeline.py b/expect_tests/pipeline.py
index 04da90309c81da86e7ef537d18c3d6e82ec0c68d..bb399c463dbfa35f3a9b2c92bf96c7a4cb30fd7b 100644
--- a/expect_tests/pipeline.py
+++ b/expect_tests/pipeline.py
@@ -3,15 +3,12 @@
 # found in the LICENSE file.
 
 import contextlib
-import ConfigParser
-import glob
 import imp
 import inspect
 import logging
 import multiprocessing
 import os
 import Queue
-import re
 import signal
 import sys
 import tempfile
@@ -26,9 +23,7 @@ from expect_tests.type_definitions import (
 
 from expect_tests.unittest_helper import _is_unittest, UnittestTestCase
 from expect_tests import util
-
-
-CONFIG_FILE_NAME = '.expect_tests.cfg'
+from expect_tests import listing
 
 
 @contextlib.contextmanager
@@ -70,38 +65,6 @@ def get_package_path(package_name, path):
   return os.path.normpath(package_path) if ispkg else None
 
 
-def get_config(path):
-  """Get configuration values
-
-  Reads the config file in provided path, and returns content.
-  See Python ConfigParser for general formatting syntax.
-
-  Example:
-  [expect_tests]
-  skip=directory1
-     directory2
-     directory3
-
-  Args:
-  path (str): path to a directory.
-
-  Returns:
-  black_list (set): blacklisted subdirectories.
-  """
-  black_list = set()
-
-  config_file_name = os.path.join(path, CONFIG_FILE_NAME)
-  parser = ConfigParser.ConfigParser()
-  parser.read([config_file_name])
-
-  if not parser.has_section('expect_tests'):
-    return black_list
-
-  if parser.has_option('expect_tests', 'skip'):
-    black_list.update(parser.get('expect_tests', 'skip').splitlines())
-  return black_list
-
-
 def is_test_file(filename):
   """Returns True if filename is supposed to contain tests.
 
@@ -114,7 +77,7 @@ def is_test_file(filename):
   return filename.endswith('_test.py')
 
 
-def walk_package(package_name, path):
+def walk_package(package_name, path, subpath=None):
   """Return all test files inside a single package.
 
   In all cases, this function returns the full package name of files ending
@@ -139,6 +102,8 @@ def walk_package(package_name, path):
   Args:
   package_name (str): name of the package, as expected by import.
   path (str): path containing the above module (optional)
+  subpath (str, optional): path inside the package, pointing to a subpackage.
+     This is used to restrict the listing to part of the package.
 
   Returns:
   test_modules (list of str): name of modules containing tests. Each element is
@@ -164,9 +129,16 @@ def walk_package(package_name, path):
 
   explored = set()
 
-  for dirpath, dirnames, filenames in os.walk(package_path, followlinks=True):
+  if subpath:
+    start_path = os.path.join(package_path, subpath)
+    if not os.path.exists(start_path):
+      raise ValueError('Provided subpath does not exist: %s' % start_path)
+  else:
+    start_path = package_path
+
+  for dirpath, dirnames, filenames in os.walk(start_path, followlinks=True):
     # Keep only submodules not blacklisted, break symlink cycles
-    blacklist = get_config(dirpath)
+    blacklist = listing.get_config(dirpath)
     dirnames[:] = [d for d in dirnames
                    if d not in blacklist and
                    os.path.isfile(os.path.join(dirpath, d, '__init__.py')) and
@@ -174,7 +146,7 @@ def walk_package(package_name, path):
     realpaths = [os.path.realpath(os.path.join(dirpath, d)) for d in dirnames]
     explored.update(realpaths)
 
-    assert dirpath.startswith(package_path)
+    assert dirpath.startswith(start_path)
     base_module_name = os.path.relpath(dirpath, base_path).split(os.path.sep)
     test_modules.extend(['.'.join(base_module_name
                                   + [inspect.getmodulename(filename)])
@@ -199,32 +171,8 @@ def load_module(modname):
   return mod
 
 
-def get_test_gens_directory(path, cwd):
-  """Given a path, return list of MultiTest or Test instances.
-
-  See UnittestTestCase for possible return values.
-
-  This function loads modules, thus no two conflicting packages (like appengine)
-  can be loaded at the same time: use separate processes for that.
-  """
-  assert isinstance(path, basestring), 'path must be a string'
-  assert os.path.isdir(path), 'path is not a directory: %s' % path
-  sys.path.insert(0, os.path.abspath(path))
-
-  test_gens = []
-  black_list = get_config(path)
-
-  for filename in filter(lambda x: x not in black_list, os.listdir(path)):
-    abs_filename = os.path.join(path, filename)
-    if (os.path.isdir(abs_filename)
-        and os.path.isfile(os.path.join(abs_filename, '__init__.py'))):
-      test_gens += get_test_gens_package(abs_filename, cwd,
-                                         update_syspath=False)
-  return test_gens
-
-
-def get_test_gens_package(package, cwd, update_syspath=True):
-  """Given a path, return list of MultiTest or Test instances.
+def get_test_gens_package(testing_context, subpath=None):
+  """Given a testing context, return list of generators of *Test instances.
 
   See UnittestTestCase for possible return values.
 
@@ -232,24 +180,21 @@ def get_test_gens_package(package, cwd, update_syspath=True):
   should be loaded at the same time: use separate processes for that.
 
   Args:
-    package (str): path to a Python package.
-    update_syspath (boolean): if True, the parent directory of 'package' is
-      prepended to sys.path.
-  """
-  assert isinstance(package, basestring), "package name should be a string."
-  assert os.path.isfile(os.path.join(package, '__init__.py')), \
-         "'package' is not pointing to a package. It must be a " + \
-         "path to a directory containing a __init__.py file."
+    testing_context (PackageTestingContext): what to test.
+    subpath (str): relative path in the tested package to restrict the search
+      to. Relative to
+      os.path.join(testing_context.cwd, testing_context.package_name)
 
+  Returns:
+    gens_list (list of generator of tests): tests are instances of Test
+      or MultiTest.
+  """
   test_gens = []
-  path = os.path.abspath(os.path.dirname(package))
-  if update_syspath:
-    sys.path.insert(0, path)
-
-  package_name = os.path.split(package.rstrip(os.path.sep))[-1]
 
-  for modname in walk_package(package_name, path):
-    with use_chdir(cwd):
+  # TODO(pgervais) add filtering on test names (use testing_context.filters)
+  for modname in walk_package(testing_context.package_name,
+                              testing_context.cwd, subpath=subpath):
+    with use_chdir(testing_context.cwd):
       mod = load_module(modname)
     for obj in mod.__dict__.values():
       if util.is_test_generator(obj):
@@ -259,89 +204,85 @@ def get_test_gens_package(package, cwd, update_syspath=True):
   return test_gens
 
 
-def gen_loop_process(gens, test_queue, result_queue, opts, kill_switch,
-                     cover_ctx, temp_dir):
+def gen_loop_process(testing_contexts, test_queue, result_queue, opts,
+                     kill_switch, cover_ctx, temp_dir):
   """Generate `Test`s from |gens|, and feed them into |test_queue|.
 
   Non-Test instances will be translated into `UnknownError` objects.
 
   Args:
-    gens: list of generators yielding Test() instances.
+    testing_contexts (list of PackageTestingContext): describe tests to
+      process.
     test_queue (multiprocessing.Queue):
     result_queue (multiprocessing.Queue):
     opts (argparse.Namespace):
     kill_switch (multiprocessing.Event):
     cover_ctx (cover.CoverageContext().create_subprocess_context)
   """
-  tempfile.tempdir = temp_dir
-
-  # Implicitly append '*' to globs that don't specify it.
-  globs = ['%s%s' % (g, '*' if '*' not in g else '') for g in opts.test_glob]
-
-  matcher = re.compile(
-      '^%s$' % '|'.join('(?:%s)' % glob.fnmatch.translate(g)
-                        for g in globs if g[0] != '-'))
-  if matcher.pattern == '^$':
-    matcher = re.compile('^.*$')
-
-  neg_matcher = re.compile(
-      '^%s$' % '|'.join('(?:%s)' % glob.fnmatch.translate(g[1:])
-                        for g in globs if g[0] == '-'))
 
   SENTINEL = object()
+  tempfile.tempdir = temp_dir
 
   def generate_tests():
-    paths_seen = set()
     seen_tests = False
     try:
-      for gen in gens:
-        gen_cover_ctx = cover_ctx(include=util.get_cover_list(gen))
-
-        with gen_cover_ctx:
-          gen_inst = gen()
-
-        while not kill_switch.is_set():
-          with gen_cover_ctx:
-            root_test = next(gen_inst, SENTINEL)
-
-          if root_test is SENTINEL:
-            break
-
-          if kill_switch.is_set():
-            break
-
-          ok_tests = []
-
-          if isinstance(root_test, MultiTest):
-            subtests = root_test.tests
-          else:
-            subtests = [root_test]
-
-          for subtest in subtests:
-            if not isinstance(subtest, Test):
-              result_queue.put_nowait(
-                  UnknownError(
-                      'Got non-[Multi]Test isinstance from generator: %r.'
-                      % subtest))
-              continue
-
-            test_path = subtest.expect_path()
-            if test_path is not None and test_path in paths_seen:
-              result_queue.put_nowait(
-                  TestError(subtest, 'Duplicate expectation path.'))
-            else:
-              if test_path is not None:
-                paths_seen.add(test_path)
-              name = subtest.name
-              if not neg_matcher.match(name) and matcher.match(name):
-                ok_tests.append(subtest)
-
-          if ok_tests:
-            seen_tests = True
-            yield root_test.restrict(ok_tests)
+      for testing_context in testing_contexts:
+        for subpath, matcher in testing_context.itermatchers():
+          paths_seen = set()
+
+          with cover_ctx:
+            gens = get_test_gens_package(testing_context, subpath=subpath)
+
+          for gen in gens:
+            gen_cover_ctx = cover_ctx(include=util.get_cover_list(gen))
+
+            with gen_cover_ctx:
+              gen_inst = gen()
+
+            while not kill_switch.is_set():
+              with gen_cover_ctx:
+                root_test = next(gen_inst, SENTINEL)
+
+              if root_test is SENTINEL:
+                break
+
+              if kill_switch.is_set():
+                break
+
+              ok_tests = []
+
+              if isinstance(root_test, MultiTest):
+                subtests = root_test.tests
+              else:
+                subtests = [root_test]
+
+              for subtest in subtests:
+                if not isinstance(subtest, Test):
+                  result_queue.put_nowait(
+                      UnknownError(
+                          'Got non-[Multi]Test isinstance from generator: %r.'
+                          % subtest))
+                  continue
+
+                test_path = subtest.expect_path()
+                if test_path is not None and test_path in paths_seen:
+                  result_queue.put_nowait(
+                      TestError(subtest, 'Duplicate expectation path.'))
+                else:
+                  if test_path is not None:
+                    paths_seen.add(test_path)
+                  name = subtest.name
+                  # if not neg_matcher.match(name) and matcher.match(name):
+                  if matcher.match(name):
+                    ok_tests.append(subtest)
+
+              if ok_tests:
+                seen_tests = True
+                yield root_test.restrict(ok_tests)
 
       if not seen_tests:
         result_queue.put_nowait(NoMatchingTestsError())
+
     except KeyboardInterrupt:
       pass
 
@@ -439,8 +380,8 @@ def run_loop_process(test_queue, result_queue, opts,
     result_queue.put_nowait)
 
 
-def result_loop_single_path(cover_ctx, kill_switch, result_queue, opts,
-                            path, path_is_package):
+def result_loop_single_context(cover_ctx, kill_switch, result_queue, opts,
+                               processing_context):
   """Run the specified operation on a single path.
 
   The path provided by the `path` argument is considered to be either a Python
@@ -458,22 +399,9 @@ def result_loop_single_path(cover_ctx, kill_switch, result_queue, opts,
     kill_switch (multiprocessing.Event):
     result_queue (multiprocessing.Queue):
     opts: output of argparse.ArgumentParser.parse_args (see main.py)
-    path (str): path a a Python package or a directory containing Python
-       packages.
-    path_is_package (boolean): tells whether 'path' is a package or not.
-  """
-  assert isinstance(path, basestring), 'path must be a string'
-
-  if path_is_package:
-    work_path = os.path.dirname(os.path.abspath(path))
-  else:
-    work_path = path
-
-  with cover_ctx:
-    if path_is_package:
-      test_gens = get_test_gens_package(path, work_path)
-    else:
-      test_gens = get_test_gens_directory(path, work_path)
+    processing_context (ProcessingContext): the task to perform.
+    """
+  sys.path.insert(0, processing_context.cwd)
 
   # This flag is set when test generation has finished.
   test_gen_finished = multiprocessing.Event()
@@ -481,7 +409,7 @@ def result_loop_single_path(cover_ctx, kill_switch, result_queue, opts,
 
   with TempDir() as temp_dir:
     test_gen_args = (
-        test_gens, test_queue, result_queue, opts,
+        processing_context.testing_contexts, test_queue, result_queue, opts,
         kill_switch, cover_ctx, temp_dir
     )
 
@@ -494,7 +422,7 @@ def result_loop_single_path(cover_ctx, kill_switch, result_queue, opts,
               target=run_loop_process,
               args=(test_queue, result_queue, opts,
                     kill_switch, test_gen_finished, cover_ctx, temp_dir,
-                    work_path),
+                    processing_context.cwd),
               name='run_loop_process %d' % job_num)
           for job_num in xrange(opts.jobs)
       ]
@@ -520,6 +448,8 @@ def result_loop(cover_ctx, opts):
   The operation to perform (list/test/debug/train) is defined by opts.handler.
   """
 
+  processing_contexts = listing.get_runtime_contexts(opts.test_glob)
+
   def ensure_echo_on():
     """Restore echo on in the terminal.
 
@@ -556,18 +486,10 @@ def result_loop(cover_ctx, opts):
 
   procs = [
     multiprocessing.Process(
-      target=result_loop_single_path,
-      args=(cover_ctx, kill_switch, result_queue, opts, os.path.abspath(p),
-            False)
-      )
-    for p in opts.directory
-    ] + [
-    multiprocessing.Process(
-      target=result_loop_single_path,
-      args=(cover_ctx, kill_switch, result_queue, opts, os.path.abspath(p),
-            True)
+      target=result_loop_single_context,
+      args=(cover_ctx, kill_switch, result_queue, opts, c)
       )
-    for p in opts.package
+    for c in processing_contexts
     ]
 
   error = False