Test infrastructure for end-to-end tests of GYP generators, and...

trunk/test/lib/TestCommon.py

+"""
+TestCommon.py:  a testing framework for commands and scripts
+                with commonly useful error handling
+
+The TestCommon module provides a simple, high-level interface for writing
+tests of executable commands and scripts, especially commands and scripts
+that interact with the file system.  All methods throw exceptions and
+exit on failure, with useful error messages.  This makes a number of
+explicit checks unnecessary, making the test scripts themselves simpler
+to write and easier to read.
+
+The TestCommon class is a subclass of the TestCmd class.  In essence,
+TestCommon is a wrapper that handles common TestCmd error conditions in
+useful ways.  You can use TestCommon directly, or subclass it for your
+program and add additional (or override) methods to tailor it to your
+program's specific needs.  Alternatively, the TestCommon class serves
+as a useful example of how to define your own TestCmd subclass.
+
+As a subclass of TestCmd, TestCommon provides access to all of the
+variables and methods from the TestCmd module.  Consequently, you can
+use any variable or method documented in the TestCmd module without
+having to explicitly import TestCmd.
+
+A TestCommon environment object is created via the usual invocation:
+
+    import TestCommon
+    test = TestCommon.TestCommon()
+
+You can use all of the TestCmd keyword arguments when instantiating a
+TestCommon object; see the TestCmd documentation for details.
+
+Here is an overview of the methods and keyword arguments that are
+provided by the TestCommon class:
+
+    test.must_be_writable('file1', ['file2', ...])
+
+    test.must_contain('file', 'required text\n')
+
+    test.must_contain_all_lines(output, lines, ['title', find])
+
+    test.must_contain_any_line(output, lines, ['title', find])
+
+    test.must_exist('file1', ['file2', ...])
+
+    test.must_match('file', "expected contents\n")
+
+    test.must_not_be_writable('file1', ['file2', ...])
+
+    test.must_not_contain_any_line(output, lines, ['title', find])
+
+    test.must_not_exist('file1', ['file2', ...])
+
+    test.run(options = "options to be prepended to arguments",
+             stdout = "expected standard output from the program",
+             stderr = "expected error output from the program",
+             status = expected_status,
+             match = match_function)
+
+The TestCommon module also provides the following variables
+
+    TestCommon.python_executable
+    TestCommon.exe_suffix
+    TestCommon.obj_suffix
+    TestCommon.shobj_prefix
+    TestCommon.shobj_suffix
+    TestCommon.lib_prefix
+    TestCommon.lib_suffix
+    TestCommon.dll_prefix
+    TestCommon.dll_suffix
+
+"""
+
+# Copyright 2000, 2001, 2002, 2003, 2004 Steven Knight
+# This module is free software, and you may redistribute it and/or modify
+# it under the same terms as Python itself, so long as this copyright message
+# and disclaimer are retained in their original form.
+#
+# IN NO EVENT SHALL THE AUTHOR BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT,
+# SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OF
+# THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+#
+# THE AUTHOR SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+# PARTICULAR PURPOSE.  THE CODE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS,
+# AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
+# SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
+
+__author__ = "Steven Knight <knight at baldmt dot com>"
+__revision__ = "TestCommon.py 0.36.D001 2009/07/24 08:45:26 knight"
+__version__ = "0.36"
+
+import copy
+import os
+import os.path
+import stat
+import string
+import sys
+import types
+import UserList
+
+from TestCmd import *
+from TestCmd import __all__
+
+__all__.extend([ 'TestCommon',
+                 'exe_suffix',
+                 'obj_suffix',
+                 'shobj_prefix',
+                 'shobj_suffix',
+                 'lib_prefix',
+                 'lib_suffix',
+                 'dll_prefix',
+                 'dll_suffix',
+               ])
+
+# Variables that describe the prefixes and suffixes on this system.
+if sys.platform == 'win32':
+    exe_suffix   = '.exe'
+    obj_suffix   = '.obj'
+    shobj_suffix = '.obj'
+    shobj_prefix = ''
+    lib_prefix   = ''
+    lib_suffix   = '.lib'
+    dll_prefix   = ''
+    dll_suffix   = '.dll'
+elif sys.platform == 'cygwin':
+    exe_suffix   = '.exe'
+    obj_suffix   = '.o'
+    shobj_suffix = '.os'
+    shobj_prefix = ''
+    lib_prefix   = 'lib'
+    lib_suffix   = '.a'
+    dll_prefix   = ''
+    dll_suffix   = '.dll'
+elif string.find(sys.platform, 'irix') != -1:
+    exe_suffix   = ''
+    obj_suffix   = '.o'
+    shobj_suffix = '.o'
+    shobj_prefix = ''
+    lib_prefix   = 'lib'
+    lib_suffix   = '.a'
+    dll_prefix   = 'lib'
+    dll_suffix   = '.so'
+elif string.find(sys.platform, 'darwin') != -1:
+    exe_suffix   = ''
+    obj_suffix   = '.o'
+    shobj_suffix = '.os'
+    shobj_prefix = ''
+    lib_prefix   = 'lib'
+    lib_suffix   = '.a'
+    dll_prefix   = 'lib'
+    dll_suffix   = '.dylib'
+elif string.find(sys.platform, 'sunos') != -1:
+    exe_suffix   = ''
+    obj_suffix   = '.o'
+    shobj_suffix = '.os'
+    shobj_prefix = 'so_'
+    lib_prefix   = 'lib'
+    lib_suffix   = '.a'
+    dll_prefix   = 'lib'
+    dll_suffix   = '.dylib'
+else:
+    exe_suffix   = ''
+    obj_suffix   = '.o'
+    shobj_suffix = '.os'
+    shobj_prefix = ''
+    lib_prefix   = 'lib'
+    lib_suffix   = '.a'
+    dll_prefix   = 'lib'
+    dll_suffix   = '.so'
+
+def is_List(e):
+    return type(e) is types.ListType \
+        or isinstance(e, UserList.UserList)
+
+def is_writable(f):
+    mode = os.stat(f)[stat.ST_MODE]
+    return mode & stat.S_IWUSR
+
+def separate_files(flist):
+    existing = []
+    missing = []
+    for f in flist:
+        if os.path.exists(f):
+            existing.append(f)
+        else:
+            missing.append(f)
+    return existing, missing
+
+if os.name == 'posix':
+    def _failed(self, status = 0):
+        if self.status is None or status is None:
+            return None
+        return _status(self) != status
+    def _status(self):
+        return self.status
+elif os.name == 'nt':
+    def _failed(self, status = 0):
+        return not (self.status is None or status is None) and \
+               self.status != status
+    def _status(self):
+        return self.status
+
+class TestCommon(TestCmd):
+
+    # Additional methods from the Perl Test::Cmd::Common module
+    # that we may wish to add in the future:
+    #
+    #  $test->subdir('subdir', ...);
+    #
+    #  $test->copy('src_file', 'dst_file');
+
+    def __init__(self, **kw):
+        """Initialize a new TestCommon instance.  This involves just
+        calling the base class initialization, and then changing directory
+        to the workdir.
+        """
+        apply(TestCmd.__init__, [self], kw)
+        os.chdir(self.workdir)
+
+    def must_be_writable(self, *files):
+        """Ensures that the specified file(s) exist and are writable.
+        An individual file can be specified as a list of directory names,
+        in which case the pathname will be constructed by concatenating
+        them.  Exits FAILED if any of the files does not exist or is
+        not writable.
+        """
+        files = map(lambda x: is_List(x) and apply(os.path.join, x) or x, files)
+        existing, missing = separate_files(files)
+        unwritable = filter(lambda x, iw=is_writable: not iw(x), existing)
+        if missing:
+            print "Missing files: `%s'" % string.join(missing, "', `")
+        if unwritable:
+            print "Unwritable files: `%s'" % string.join(unwritable, "', `")
+        self.fail_test(missing + unwritable)
+
+    def must_contain(self, file, required, mode = 'rb'):
+        """Ensures that the specified file contains the required text.
+        """
+        file_contents = self.read(file, mode)
+        contains = (string.find(file_contents, required) != -1)
+        if not contains:
+            print "File `%s' does not contain required string." % file
+            print self.banner('Required string ')
+            print required
+            print self.banner('%s contents ' % file)
+            print file_contents
+            self.fail_test(not contains)
+
+    def must_contain_all_lines(self, output, lines, title=None, find=None):
+        """Ensures that the specified output string (first argument)
+        contains all of the specified lines (second argument).
+
+        An optional third argument can be used to describe the type
+        of output being searched, and only shows up in failure output.
+
+        An optional fourth argument can be used to supply a different
+        function, of the form "find(line, output), to use when searching
+        for lines in the output.
+        """
+        if find is None:
+            find = lambda o, l: string.find(o, l) != -1
+        missing = []
+        for line in lines:
+            if not find(output, line):
+                missing.append(line)
+
+        if missing:
+            if title is None:
+                title = 'output'
+            sys.stdout.write("Missing expected lines from %s:\n" % title)
+            for line in missing:
+                sys.stdout.write('    ' + repr(line) + '\n')
+            sys.stdout.write(self.banner(title + ' '))
+            sys.stdout.write(output)
+            self.fail_test()
+
+    def must_contain_any_line(self, output, lines, title=None, find=None):
+        """Ensures that the specified output string (first argument)
+        contains at least one of the specified lines (second argument).
+
+        An optional third argument can be used to describe the type
+        of output being searched, and only shows up in failure output.
+
+        An optional fourth argument can be used to supply a different
+        function, of the form "find(line, output), to use when searching
+        for lines in the output.
+        """
+        if find is None:
+            find = lambda o, l: string.find(o, l) != -1
+        for line in lines:
+            if find(output, line):
+                return
+
+        if title is None:
+            title = 'output'
+        sys.stdout.write("Missing any expected line from %s:\n" % title)
+        for line in lines:
+            sys.stdout.write('    ' + repr(line) + '\n')
+        sys.stdout.write(self.banner(title + ' '))
+        sys.stdout.write(output)
+        self.fail_test()
+
+    def must_contain_lines(self, lines, output, title=None):
+        # Deprecated; retain for backwards compatibility.
+        return self.must_contain_all_lines(output, lines, title)
+
+    def must_exist(self, *files):
+        """Ensures that the specified file(s) must exist.  An individual
+        file be specified as a list of directory names, in which case the
+        pathname will be constructed by concatenating them.  Exits FAILED
+        if any of the files does not exist.
+        """
+        files = map(lambda x: is_List(x) and apply(os.path.join, x) or x, files)
+        missing = filter(lambda x: not os.path.exists(x), files)
+        if missing:
+            print "Missing files: `%s'" % string.join(missing, "', `")
+            self.fail_test(missing)
+
+    def must_match(self, file, expect, mode = 'rb'):
+        """Matches the contents of the specified file (first argument)
+        against the expected contents (second argument).  The expected
+        contents are a list of lines or a string which will be split
+        on newlines.
+        """
+        file_contents = self.read(file, mode)
+        try:
+            self.fail_test(not self.match(file_contents, expect))
+        except KeyboardInterrupt:
+            raise
+        except:
+            print "Unexpected contents of `%s'" % file
+            self.diff(expect, file_contents, 'contents ')
+            raise
+
+    def must_not_contain_any_line(self, output, lines, title=None, find=None):
+        """Ensures that the specified output string (first argument)
+        does not contain any of the specified lines (second argument).
+
+        An optional third argument can be used to describe the type
+        of output being searched, and only shows up in failure output.
+
+        An optional fourth argument can be used to supply a different
+        function, of the form "find(line, output), to use when searching
+        for lines in the output.
+        """
+        if find is None:
+            find = lambda o, l: string.find(o, l) != -1
+        unexpected = []
+        for line in lines:
+            if find(output, line):
+                unexpected.append(line)
+
+        if unexpected:
+            if title is None:
+                title = 'output'
+            sys.stdout.write("Unexpected lines in %s:\n" % title)
+            for line in unexpected:
+                sys.stdout.write('    ' + repr(line) + '\n')
+            sys.stdout.write(self.banner(title + ' '))
+            sys.stdout.write(output)
+            self.fail_test()
+
+    def must_not_contain_lines(self, lines, output, title=None):
+        return self.must_not_contain_any_line(output, lines, title)
+
+    def must_not_exist(self, *files):
+        """Ensures that the specified file(s) must not exist.
+        An individual file be specified as a list of directory names, in
+        which case the pathname will be constructed by concatenating them.
+        Exits FAILED if any of the files exists.
+        """
+        files = map(lambda x: is_List(x) and apply(os.path.join, x) or x, files)
+        existing = filter(os.path.exists, files)
+        if existing:
+            print "Unexpected files exist: `%s'" % string.join(existing, "', `")
+            self.fail_test(existing)
+
+
+    def must_not_be_writable(self, *files):
+        """Ensures that the specified file(s) exist and are not writable.
+        An individual file can be specified as a list of directory names,
+        in which case the pathname will be constructed by concatenating
+        them.  Exits FAILED if any of the files does not exist or is
+        writable.
+        """
+        files = map(lambda x: is_List(x) and apply(os.path.join, x) or x, files)
+        existing, missing = separate_files(files)
+        writable = filter(is_writable, existing)
+        if missing:
+            print "Missing files: `%s'" % string.join(missing, "', `")
+        if writable:
+            print "Writable files: `%s'" % string.join(writable, "', `")
+        self.fail_test(missing + writable)
+
+    def _complete(self, actual_stdout, expected_stdout,
+                        actual_stderr, expected_stderr, status, match):
+        """
+        Post-processes running a subcommand, checking for failure
+        status and displaying output appropriately.
+        """
+        if _failed(self, status):
+            expect = ''
+            if status != 0:
+                expect = " (expected %s)" % str(status)
+            print "%s returned %s%s" % (self.program, str(_status(self)), expect)
+            print self.banner('STDOUT ')
+            print actual_stdout
+            print self.banner('STDERR ')
+            print actual_stderr
+            self.fail_test()
+        if not expected_stdout is None and not match(actual_stdout, expected_stdout):
+            self.diff(expected_stdout, actual_stdout, 'STDOUT ')
+            if actual_stderr:
+                print self.banner('STDERR ')
+                print actual_stderr
+            self.fail_test()
+        if not expected_stderr is None and not match(actual_stderr, expected_stderr):
+            print self.banner('STDOUT ')
+            print actual_stdout
+            self.diff(expected_stderr, actual_stderr, 'STDERR ')
+            self.fail_test()
+
+    def start(self, program = None,
+                    interpreter = None,
+                    arguments = None,
+                    universal_newlines = None,
+                    **kw):
+        """
+        Starts a program or script for the test environment.
+
+        This handles the "options" keyword argument and exceptions.
+        """
+        try:
+            options = kw['options']
+            del kw['options']
+        except KeyError:
+            pass
+        else:
+            if options:
+                if arguments is None:
+                    arguments = options
+                else:
+                    arguments = options + " " + arguments
+        try:
+            return apply(TestCmd.start,
+                         (self, program, interpreter, arguments, universal_newlines),
+                         kw)
+        except KeyboardInterrupt:
+            raise
+        except Exception, e:
+            print self.banner('STDOUT ')
+            try:
+                print self.stdout()
+            except IndexError:
+                pass
+            print self.banner('STDERR ')
+            try:
+                print self.stderr()
+            except IndexError:
+                pass
+            cmd_args = self.command_args(program, interpreter, arguments)
+            sys.stderr.write('Exception trying to execute: %s\n' % cmd_args)
+            raise e
+
+    def finish(self, popen, stdout = None, stderr = '', status = 0, **kw):
+        """
+        Finishes and waits for the process being run under control of
+        the specified popen argument.  Additional arguments are similar
+        to those of the run() method:
+
+                stdout  The expected standard output from
+                        the command.  A value of None means
+                        don't test standard output.
+
+                stderr  The expected error output from
+                        the command.  A value of None means
+                        don't test error output.
+
+                status  The expected exit status from the
+                        command.  A value of None means don't
+                        test exit status.
+        """
+        apply(TestCmd.finish, (self, popen,), kw)
+        match = kw.get('match', self.match)
+        self._complete(self.stdout(), stdout,
+                       self.stderr(), stderr, status, match)
+
+    def run(self, options = None, arguments = None,
+                  stdout = None, stderr = '', status = 0, **kw):
+        """Runs the program under test, checking that the test succeeded.
+
+        The arguments are the same as the base TestCmd.run() method,
+        with the addition of:
+
+                options Extra options that get appended to the beginning
+                        of the arguments.
+
+                stdout  The expected standard output from
+                        the command.  A value of None means
+                        don't test standard output.
+
+                stderr  The expected error output from
+                        the command.  A value of None means
+                        don't test error output.
+
+                status  The expected exit status from the
+                        command.  A value of None means don't
+                        test exit status.
+
+        By default, this expects a successful exit (status = 0), does
+        not test standard output (stdout = None), and expects that error
+        output is empty (stderr = "").
+        """
+        if options:
+            if arguments is None:
+                arguments = options
+            else:
+                arguments = options + " " + arguments
+        kw['arguments'] = arguments
+        try:
+            match = kw['match']
+            del kw['match']
+        except KeyError:
+            match = self.match
+        apply(TestCmd.run, [self], kw)
+        self._complete(self.stdout(), stdout,
+                       self.stderr(), stderr, status, match)
+
+    def skip_test(self, message="Skipping test.\n"):
+        """Skips a test.
+
+        Proper test-skipping behavior is dependent on the external
+        TESTCOMMON_PASS_SKIPS environment variable.  If set, we treat
+        the skip as a PASS (exit 0), and otherwise treat it as NO RESULT.
+        In either case, we print the specified message as an indication
+        that the substance of the test was skipped.
+
+        (This was originally added to support development under Aegis.
+        Technically, skipping a test is a NO RESULT, but Aegis would
+        treat that as a test failure and prevent the change from going to
+        the next step.  Since we ddn't want to force anyone using Aegis
+        to have to install absolutely every tool used by the tests, we
+        would actually report to Aegis that a skipped test has PASSED
+        so that the workflow isn't held up.)
+        """
+        if message:
+            sys.stdout.write(message)
+            sys.stdout.flush()
+        pass_skips = os.environ.get('TESTCOMMON_PASS_SKIPS')
+        if pass_skips in [None, 0, '0']:
+            # skip=1 means skip this function when showing where this
+            # result came from.  They only care about the line where the
+            # script called test.skip_test(), not the line number where
+            # we call test.no_result().
+            self.no_result(skip=1)
+        else:
+            # We're under the development directory for this change,
+            # so this is an Aegis invocation; pass the test (exit 0).
+            self.pass_test()
+
+# Local Variables:
+# tab-width:4
+# indent-tabs-mode:nil
+# End:
+# vim: set expandtab tabstop=4 shiftwidth=4: