New expect_tests UI

expect_tests/test/pipeline_test.py

Index: expect_tests/test/pipeline_test.py
diff --git a/expect_tests/test/pipeline_test.py b/expect_tests/test/pipeline_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..c1c2512432261111b9a053b969b8200f1d011a0f
--- /dev/null
+++ b/expect_tests/test/pipeline_test.py
@@ -0,0 +1,322 @@
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+import sys
+import unittest
+
+from expect_tests import pipeline
+from expect_tests import listing
+from expect_tests.type_definitions import Test, MultiTest
+
+SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__))
+DATA_DIR = os.path.join(SCRIPT_DIR, 'data')
+
+class ParsingAndConfigTest(unittest.TestCase):
+  def test_get_config(self):
+    """Testing that reading the config file works.
+
+    Tests requires a specific content for data/.expect_tests.cfg"""
+    black_list = listing.get_config(DATA_DIR)
+    self.assertEqual(black_list,
+                     set(['directory1', 'directory2', 'ignored']))
+
+  def test_parse_test_glob(self):
+    self.assertEqual(listing.parse_test_glob('a/b/c'),
+                     (os.path.abspath('a/b/c'), ('*',)))
+    self.assertEqual(listing.parse_test_glob('a/b/c:'),
+                     (os.path.abspath('a/b/c'), ('*',)))
+    self.assertEqual(listing.parse_test_glob('a/b/c:Test'),
+                     (os.path.abspath('a/b/c'), 'Test'))
+    self.assertEqual(listing.parse_test_glob('a/b/c:Test.Name'),
+                     (os.path.abspath('a/b/c'), 'Test.Name'))
+    self.assertRaises(ValueError, listing.parse_test_glob, 'a:b:c',)
+    self.assertRaises(ValueError, listing.parse_test_glob, 'a:b/c',)
+
+
+class PathManipulationTest(unittest.TestCase):
+  """Tests for all path-manipulating functions.
+
+  This set uses checked-out files in the present repository to avoid mocking
+  the I/O functions.
+  """
+
+  def test_get_python_root(self):
+    """This function uses the directory structure under data/"""
+
+    cases = [
+      # The root of a directory with no __init__.py file is that directory
+      (DATA_DIR, DATA_DIR),
+      # The root of a package is the parent directory
+      (os.path.join(DATA_DIR, 'package1'), DATA_DIR),
+      # The root of a subpackage is the parent directory of the root package.
+      (os.path.join(DATA_DIR, 'package1', 'subpackage1_1'), DATA_DIR)
+      ]
+    for path, result in cases:
+      self.assertEqual(listing.get_python_root(path), result)
+
+    # When the path does not exist, you get an error.
+    self.assertRaises(ValueError, listing.get_python_root,
+                      '____non-existing-path____')
+
+  def test_single_dir_runtime_context(self):
+    """Computing RuntimeContext from a single directory path."""
+    test_globs = [DATA_DIR]
+    contexts = listing.get_runtime_contexts(test_globs)
+    self.assertEqual(len(contexts), 1)
+    self.assertEqual(contexts[0].cwd, DATA_DIR)
+    for testing_c in contexts[0].testing_contexts:
+      self.assertNotEqual(testing_c.package_name, 'ignored')
+
+  def test_single_package_runtime_context(self):
+    """Computing RuntimeContext from a single package path."""
+    test_globs = [os.path.join(DATA_DIR, 'package1')]
+    contexts = listing.get_runtime_contexts(test_globs)
+    self.assertEqual(len(contexts), 1)
+    self.assertEqual(contexts[0].cwd, DATA_DIR)
+    testing_c = contexts[0].testing_contexts
+    self.assertEqual(len(testing_c), 1)
+    self.assertEqual(testing_c[0].package_name, 'package1')
+
+  def test_two_packages_runtime_context(self):
+    """Computing RuntimeContext from two package paths."""
+    test_globs = [os.path.join(DATA_DIR, 'package1'),
+                  os.path.join(DATA_DIR, 'package2')]
+    contexts = listing.get_runtime_contexts(test_globs)
+    self.assertEqual(len(contexts), 1)
+    self.assertEqual(contexts[0].cwd, DATA_DIR)
+    testing_c = contexts[0].testing_contexts
+    self.assertEqual(len(testing_c), 2)
+    package_names = set()
+    for testing_c in contexts[0].testing_contexts:
+      package_names.add(testing_c.package_name)
+
+    self.assertEqual(package_names, set(('package1', 'package2')))
+
+  def test_package_and_directory_runtime_context(self):
+    """Computing RuntimeContext from a package and a directory paths.
+    """
+    # 'package1' is specified both explicit and implicitly through DATA_DIR
+    # We check that it is accounted for only once.
+    test_globs = [DATA_DIR, os.path.join(DATA_DIR, 'package1')]
+    contexts = listing.get_runtime_contexts(test_globs)
+    self.assertEqual(len(contexts), 1)
+    # 2 is the number of packages under DATA_DIR, not counting
+    # the ignored ones.
+    self.assertEqual(len(contexts[0].testing_contexts), 2)
+
+    test_globs = [DATA_DIR, os.path.join(DATA_DIR, 'package1'),
+                  os.path.join(DATA_DIR, 'package1', 'subpackage1_1')]
+    contexts = listing.get_runtime_contexts(test_globs)
+    self.assertEqual(len(contexts), 1)
+    self.assertEqual(len(contexts[0].testing_contexts), 2)
+
+  def test_package_testing_context_from_path(self):

[agable, 2014/11/14 20:00:09]

I still think this should be three methods:
test_package_testing_context_no_filters
""_empty_filters
""_with_filters

I haven't seen any reason *not* to break it up, and doing so lets more granular
results be reported.

+    """Test the PackageTestingContext class"""
+    package_name = 'package1'
+    package1 = os.path.join(SCRIPT_DIR, 'data', package_name)
+
+    context = listing.PackageTestingContext.from_path(package1)
+    self.assertTrue(os.path.isabs(context.cwd))
+    self.assertTrue(len(context.package_name) > 0)
+    self.assertEqual(len(context.filters), 1)
+    self.assertEqual(context.filters[0][1], '*')
+
+    # Testing with an empty tuple.
+    context = listing.PackageTestingContext.from_path(package1,
+                                                       filters=())
+    self.assertTrue(os.path.isabs(context.cwd))
+    self.assertTrue(len(context.package_name) > 0)
+    self.assertEqual(len(context.filters), 1)
+    self.assertEqual(context.filters[0][1], '*')
+
+    context = listing.PackageTestingContext.from_path(package1,
+                                                       filters=('a*', 'b*'))
+    self.assertTrue(os.path.isabs(context.cwd))
+    self.assertTrue(len(context.package_name) > 0)
+    self.assertEqual(len(context.filters), 2)
+    self.assertEqual(context.filters[0][1], 'a*')
+    self.assertEqual(context.filters[1][1], 'b*')
+
+    self.assertRaises(ValueError,
+                      listing.PackageTestingContext.from_path,
+                      package1, filters=None)
+
+  def test_merging_package_testing_context(self):
+    """Merging PackageTestingContexts pointing at the same package.

[agable, 2014/11/14 20:00:09]

Four methods. In the current method, it's not even clear *why* the second and
third test cases are supposed to raise!
test_merging_package_testing_context_same_package
test_merging_package_testing_context_???
test_merging_simlar_package_testing_context_raises
test_merging_package_and_subpackage_testing_contexts

+    """
+    package_name = 'package1'
+    package1 = os.path.join(SCRIPT_DIR, 'data', package_name)
+    package2 = os.path.join(SCRIPT_DIR, 'data', 'package2')
+    other_package1 = os.path.join(SCRIPT_DIR, 'data', 'other', package_name)
+
+    # from_context_list
+    c1 = listing.PackageTestingContext.from_path(package1, filters=('a',))
+    c2 = listing.PackageTestingContext.from_path(package1,
+                                                      filters=('b','c'))
+    context = listing.PackageTestingContext.from_context_list((c1, c2))
+    self.assertEqual(len(context.filters), 3)
+    self.assertEqual(set(filt[1] for filt in context.filters),
+                     set(('a', 'b', 'c')))
+
+    c1 = listing.PackageTestingContext.from_path(package1, filters=('a',))
+    c2 = listing.PackageTestingContext.from_path(package2,
+                                                      filters=('b','c'))
+    self.assertRaises(ValueError,
+                      listing.PackageTestingContext.from_context_list,
+                      (c1, c2))
+
+    # Same package name, different paths.
+    c1 = listing.PackageTestingContext.from_path(package1, filters=('a',))
+    c2 = listing.PackageTestingContext.from_path(other_package1,
+                                                  filters=('b','c'))
+    self.assertRaises(ValueError,
+                      listing.PackageTestingContext.from_context_list,
+                      (c1, c2))
+
+    # Subpackage
+    subpackage_path = 'subpackage1_1'
+    subpackage1 = os.path.join(package1, subpackage_path)
+    c1 = listing.PackageTestingContext.from_path(subpackage1)
+    self.assertEqual(c1.package_name, 'package1')
+    self.assertEqual(c1.filters, [(subpackage_path, '*')])
+
+  def test_filter_glob_manipulation(self):

[agable, 2014/11/14 20:00:09]

test_filter_glob_respects_star
test_filter_glob_adds_implied_star

+    """globs to filter tests are modified if they don't end with a *."""
+    package_name = 'package1'
+    package1 = os.path.join(SCRIPT_DIR, 'data', package_name)
+    subpackage_path = 'subpackage1_1'
+    subpackage1 = os.path.join(package1, subpackage_path)
+    c1 = listing.PackageTestingContext.from_path(subpackage1,
+                                                  filters=('a*',))

[agable, 2014/11/14 20:00:09]

nit: indentation

+    self.assertEqual(c1.package_name, 'package1')
+    self.assertEqual(c1.filters, [(subpackage_path, 'a*')])
+
+    for subpath, matcher in c1.itermatchers():
+      print matcher.pattern

[agable, 2014/11/14 20:00:09]

left a debugging print in?

+      self.assertIsNotNone(matcher.match('a'))
+      self.assertIsNotNone(matcher.match('ab'))
+      self.assertIsNone(matcher.match('ba'))
+      self.assertIsNone(matcher.match('b'))
+      self.assertEqual(subpath, subpackage_path)
+
+    # Test that a star is added to the filter.
+    c1 = listing.PackageTestingContext.from_path(subpackage1,
+                                                      filters=('a',))

[agable, 2014/11/14 20:00:09]

nit: intendation

+    self.assertEqual(c1.package_name, 'package1')
+    self.assertEqual(c1.filters, [(subpackage_path, 'a')])
+
+    for subpath, matcher in c1.itermatchers():
+      print matcher.pattern
+      self.assertIsNotNone(matcher.match('a'))
+      self.assertIsNotNone(matcher.match('ab'))
+      self.assertIsNone(matcher.match('ba'))
+      self.assertIsNone(matcher.match('b'))
+      self.assertEqual(subpath, subpackage_path)
+
+
+  def test_processing_context(self):
+    """Test the ProcessingContext class"""
+    package_name = 'package1'
+    package1 = os.path.join(SCRIPT_DIR, 'data', package_name)
+    subpackage1 = os.path.join(SCRIPT_DIR, 'data',
+                               package_name, 'subpackage1_1')
+    package2 = os.path.join(SCRIPT_DIR, 'data', 'package2')
+    other_package1 = os.path.join(SCRIPT_DIR, 'data', 'other', package_name)
+
+    c0 = listing.PackageTestingContext.from_path(package1)
+    c1 = listing.PackageTestingContext.from_path(package1, filters=('a',))
+    c2 = listing.PackageTestingContext.from_path(subpackage1,
+                                                      filters=('d',))
+    c3 = listing.PackageTestingContext.from_path(package2,
+                                                      filters=('b','c'))
+    c4 = listing.PackageTestingContext.from_path(other_package1)
+
+    # A processing context is a cwd + testing contexts.
+    # A testing context is cwd + one package name.
+    context = listing.ProcessingContext((c1, c2))
+    self.assertEqual(len(context.testing_contexts), 1)
+    self.assertEqual(set(filt[1]
+                         for filt in context.testing_contexts[0].filters),
+                     set(('a', 'd')))
+
+    context = listing.ProcessingContext((c0, c1, c2))
+    self.assertEqual(len(context.testing_contexts), 1)
+    self.assertEqual(set(filt[1]
+                         for filt in context.testing_contexts[0].filters),
+                     set(('*', 'a', 'd')))
+
+
+    context = listing.ProcessingContext((c1, c2, c3))
+    self.assertEqual(len(context.testing_contexts), 2)
+
+    # Fails because there are two different cwd.

[agable, 2014/11/14 20:00:09]

test_processing_context_requires_same_cwd

+    self.assertRaises(ValueError, listing.ProcessingContext, (c1, c4))
+
+
+class TestListingTest(unittest.TestCase):
+  """Test functions related to listing tests."""
+  def test_walk_package(self):
+    """This function uses the directory structure under data/"""
+    modules = pipeline.walk_package('package1', DATA_DIR)
+    self.assertEqual(modules,
+                     ['package1.file1_test', 'package1.file2_test',
+                      'package1.subpackage1_1.file3_test',
+                      'package1.subpackage1_1.subpackage1_1_1.file4_test'])
+
+    modules = pipeline.walk_package('package1', DATA_DIR,
+                                    subpath='subpackage1_1')
+    self.assertEqual(modules,
+                     ['package1.subpackage1_1.file3_test',
+                      'package1.subpackage1_1.subpackage1_1_1.file4_test'])
+
+    self.assertRaises(ValueError, pipeline.walk_package,
+                      'package1', DATA_DIR, 'non-existing')
+
+  def test_get_test_gens_package(self):
+    def get_test_names(tests):
+      test_names = []
+
+      for gen in tests:
+        for test in gen():
+          if isinstance(test, MultiTest):
+            subtests = test.tests
+          else:
+            subtests = [test]
+
+          for subtest in subtests:
+            self.assertIsInstance(subtest, Test)
+            test_names.append(subtest.name)
+      return test_names
+
+    sys.path.insert(0, DATA_DIR)  # Ugh. But won't work otherwise.
+
+
+    # TODO(pgervais): add a MultiTest in a package under data/
+    package1 = os.path.join(DATA_DIR, 'package1')
+    testing_context = listing.PackageTestingContext.from_path(package1)
+    tests = pipeline.get_test_gens_package(testing_context)
+    self.assertEqual(
+      get_test_names(tests),
+      ['package1.file1_test.File1Test.test_trivial_1',
+       'package1.file2_test.File2Test.test_trivial_2',
+       'package1.subpackage1_1.file3_test.File3Test.test_trivial_3',
+       'package1.subpackage1_1.subpackage1_1_1.file4_test.' +
+           'File4Test.test_trivial_4'])
+
+    tests = pipeline.get_test_gens_package(
+      testing_context, subpath='subpackage1_1')
+    self.assertEqual(
+      get_test_names(tests),
+      ['package1.subpackage1_1.file3_test.File3Test.test_trivial_3',
+       'package1.subpackage1_1.subpackage1_1_1.file4_test.' +
+           'File4Test.test_trivial_4'])
+
+    tests = pipeline.get_test_gens_package(
+      testing_context, subpath='subpackage1_1/subpackage1_1_1')
+
+    self.assertEqual(
+      get_test_names(tests),
+      ['package1.subpackage1_1.subpackage1_1_1.file4_test.' +
+       'File4Test.test_trivial_4'])