OLD | NEW |
(Empty) | |
| 1 # -*- coding: utf-8 -*- |
| 2 |
| 3 """unittest-xml-reporting is a PyUnit-based TestRunner that can export test |
| 4 results to XML files that can be consumed by a wide range of tools, such as |
| 5 build systems, IDEs and Continuous Integration servers. |
| 6 |
| 7 This module provides the XMLTestRunner class, which is heavily based on the |
| 8 default TextTestRunner. This makes the XMLTestRunner very simple to use. |
| 9 |
| 10 The script below, adapted from the unittest documentation, shows how to use |
| 11 XMLTestRunner in a very simple way. In fact, the only difference between this |
| 12 script and the original one is the last line: |
| 13 |
| 14 import random |
| 15 import unittest |
| 16 import xmlrunner |
| 17 |
| 18 class TestSequenceFunctions(unittest.TestCase): |
| 19 def setUp(self): |
| 20 self.seq = range(10) |
| 21 |
| 22 def test_shuffle(self): |
| 23 # make sure the shuffled sequence does not lose any elements |
| 24 random.shuffle(self.seq) |
| 25 self.seq.sort() |
| 26 self.assertEqual(self.seq, range(10)) |
| 27 |
| 28 def test_choice(self): |
| 29 element = random.choice(self.seq) |
| 30 self.assert_(element in self.seq) |
| 31 |
| 32 def test_sample(self): |
| 33 self.assertRaises(ValueError, random.sample, self.seq, 20) |
| 34 for element in random.sample(self.seq, 5): |
| 35 self.assert_(element in self.seq) |
| 36 |
| 37 if __name__ == '__main__': |
| 38 unittest.main(testRunner=xmlrunner.XMLTestRunner(output='test-reports')) |
| 39 """ |
| 40 |
| 41 import os |
| 42 import sys |
| 43 import time |
| 44 from unittest import TestResult, _TextTestResult, TextTestRunner |
| 45 from cStringIO import StringIO |
| 46 import xml.dom.minidom |
| 47 |
| 48 |
| 49 class XMLDocument(xml.dom.minidom.Document): |
| 50 def createCDATAOrText(self, data): |
| 51 if ']]>' in data: |
| 52 return self.createTextNode(data) |
| 53 return self.createCDATASection(data) |
| 54 |
| 55 |
| 56 class _TestInfo(object): |
| 57 """This class is used to keep useful information about the execution of a |
| 58 test method. |
| 59 """ |
| 60 |
| 61 # Possible test outcomes |
| 62 (SUCCESS, FAILURE, ERROR) = range(3) |
| 63 |
| 64 def __init__(self, test_result, test_method, outcome=SUCCESS, err=None): |
| 65 "Create a new instance of _TestInfo." |
| 66 self.test_result = test_result |
| 67 self.test_method = test_method |
| 68 self.outcome = outcome |
| 69 self.err = err |
| 70 self.stdout = test_result.stdout and test_result.stdout.getvalue().strip
() or '' |
| 71 self.stderr = test_result.stdout and test_result.stderr.getvalue().strip
() or '' |
| 72 |
| 73 def get_elapsed_time(self): |
| 74 """Return the time that shows how long the test method took to |
| 75 execute. |
| 76 """ |
| 77 return self.test_result.stop_time - self.test_result.start_time |
| 78 |
| 79 def get_description(self): |
| 80 "Return a text representation of the test method." |
| 81 return self.test_result.getDescription(self.test_method) |
| 82 |
| 83 def get_error_info(self): |
| 84 """Return a text representation of an exception thrown by a test |
| 85 method. |
| 86 """ |
| 87 if not self.err: |
| 88 return '' |
| 89 if sys.version_info < (2,4): |
| 90 return self.test_result._exc_info_to_string(self.err) |
| 91 else: |
| 92 return self.test_result._exc_info_to_string( |
| 93 self.err, self.test_method) |
| 94 |
| 95 |
| 96 class _XMLTestResult(_TextTestResult): |
| 97 """A test result class that can express test results in a XML report. |
| 98 |
| 99 Used by XMLTestRunner. |
| 100 """ |
| 101 def __init__(self, stream=sys.stderr, descriptions=1, verbosity=1, \ |
| 102 elapsed_times=True): |
| 103 "Create a new instance of _XMLTestResult." |
| 104 _TextTestResult.__init__(self, stream, descriptions, verbosity) |
| 105 self.successes = [] |
| 106 self.callback = None |
| 107 self.elapsed_times = elapsed_times |
| 108 self.output_patched = False |
| 109 |
| 110 def _prepare_callback(self, test_info, target_list, verbose_str, |
| 111 short_str): |
| 112 """Append a _TestInfo to the given target list and sets a callback |
| 113 method to be called by stopTest method. |
| 114 """ |
| 115 target_list.append(test_info) |
| 116 def callback(): |
| 117 """This callback prints the test method outcome to the stream, |
| 118 as well as the elapsed time. |
| 119 """ |
| 120 |
| 121 # Ignore the elapsed times for a more reliable unit testing |
| 122 if not self.elapsed_times: |
| 123 self.start_time = self.stop_time = 0 |
| 124 |
| 125 if self.showAll: |
| 126 self.stream.writeln('(%.3fs) %s' % \ |
| 127 (test_info.get_elapsed_time(), verbose_str)) |
| 128 elif self.dots: |
| 129 self.stream.write(short_str) |
| 130 self.callback = callback |
| 131 |
| 132 def _patch_standard_output(self): |
| 133 """Replace the stdout and stderr streams with string-based streams |
| 134 in order to capture the tests' output. |
| 135 """ |
| 136 if not self.output_patched: |
| 137 (self.old_stdout, self.old_stderr) = (sys.stdout, sys.stderr) |
| 138 self.output_patched = True |
| 139 (sys.stdout, sys.stderr) = (self.stdout, self.stderr) = \ |
| 140 (StringIO(), StringIO()) |
| 141 |
| 142 def _restore_standard_output(self): |
| 143 "Restore the stdout and stderr streams." |
| 144 (sys.stdout, sys.stderr) = (self.old_stdout, self.old_stderr) |
| 145 self.output_patched = False |
| 146 |
| 147 def startTest(self, test): |
| 148 "Called before execute each test method." |
| 149 self._patch_standard_output() |
| 150 self.start_time = time.time() |
| 151 TestResult.startTest(self, test) |
| 152 |
| 153 if self.showAll: |
| 154 self.stream.write(' ' + self.getDescription(test)) |
| 155 self.stream.write(" ... ") |
| 156 |
| 157 def stopTest(self, test): |
| 158 "Called after execute each test method." |
| 159 self._restore_standard_output() |
| 160 _TextTestResult.stopTest(self, test) |
| 161 self.stop_time = time.time() |
| 162 |
| 163 if self.callback and callable(self.callback): |
| 164 self.callback() |
| 165 self.callback = None |
| 166 |
| 167 def addSuccess(self, test): |
| 168 "Called when a test executes successfully." |
| 169 self._prepare_callback(_TestInfo(self, test), |
| 170 self.successes, 'OK', '.') |
| 171 |
| 172 def addFailure(self, test, err): |
| 173 "Called when a test method fails." |
| 174 self._prepare_callback(_TestInfo(self, test, _TestInfo.FAILURE, err), |
| 175 self.failures, 'FAIL', 'F') |
| 176 |
| 177 def addError(self, test, err): |
| 178 "Called when a test method raises an error." |
| 179 self._prepare_callback(_TestInfo(self, test, _TestInfo.ERROR, err), |
| 180 self.errors, 'ERROR', 'E') |
| 181 |
| 182 def printErrorList(self, flavour, errors): |
| 183 "Write some information about the FAIL or ERROR to the stream." |
| 184 for test_info in errors: |
| 185 if isinstance(test_info, tuple): |
| 186 test_info, exc_info = test_info |
| 187 self.stream.writeln(self.separator1) |
| 188 self.stream.writeln('%s [%.3fs]: %s' % ( |
| 189 flavour, test_info.get_elapsed_time(), |
| 190 test_info.get_description())) |
| 191 self.stream.writeln(self.separator2) |
| 192 self.stream.writeln('%s' % test_info.get_error_info()) |
| 193 |
| 194 def _get_info_by_testcase(self): |
| 195 """This method organizes test results by TestCase module. This |
| 196 information is used during the report generation, where a XML report |
| 197 will be generated for each TestCase. |
| 198 """ |
| 199 tests_by_testcase = {} |
| 200 |
| 201 for tests in (self.successes, self.failures, self.errors): |
| 202 for test_info in tests: |
| 203 testcase = type(test_info.test_method) |
| 204 |
| 205 # Ignore module name if it is '__main__' |
| 206 module = testcase.__module__ + '.' |
| 207 if module == '__main__.': |
| 208 module = '' |
| 209 testcase_name = module + testcase.__name__ |
| 210 |
| 211 if testcase_name not in tests_by_testcase: |
| 212 tests_by_testcase[testcase_name] = [] |
| 213 tests_by_testcase[testcase_name].append(test_info) |
| 214 |
| 215 return tests_by_testcase |
| 216 |
| 217 def _report_testsuite(suite_name, tests, xml_document): |
| 218 "Appends the testsuite section to the XML document." |
| 219 testsuite = xml_document.createElement('testsuite') |
| 220 xml_document.appendChild(testsuite) |
| 221 |
| 222 testsuite.setAttribute('name', str(suite_name)) |
| 223 testsuite.setAttribute('tests', str(len(tests))) |
| 224 |
| 225 testsuite.setAttribute('time', '%.3f' % |
| 226 sum([e.get_elapsed_time() for e in tests])) |
| 227 |
| 228 failures = len([1 for e in tests if e.outcome == _TestInfo.FAILURE]) |
| 229 testsuite.setAttribute('failures', str(failures)) |
| 230 |
| 231 errors = len([1 for e in tests if e.outcome == _TestInfo.ERROR]) |
| 232 testsuite.setAttribute('errors', str(errors)) |
| 233 |
| 234 return testsuite |
| 235 |
| 236 _report_testsuite = staticmethod(_report_testsuite) |
| 237 |
| 238 def _report_testcase(suite_name, test_result, xml_testsuite, xml_document): |
| 239 "Appends a testcase section to the XML document." |
| 240 testcase = xml_document.createElement('testcase') |
| 241 xml_testsuite.appendChild(testcase) |
| 242 |
| 243 testcase.setAttribute('classname', str(suite_name)) |
| 244 testcase.setAttribute('name', test_result.test_method.shortDescription() |
| 245 or getattr(test_result.test_method, '_testMethodNa
me', |
| 246 str(test_result.test_method))) |
| 247 testcase.setAttribute('time', '%.3f' % test_result.get_elapsed_time()) |
| 248 |
| 249 if (test_result.outcome != _TestInfo.SUCCESS): |
| 250 elem_name = ('failure', 'error')[test_result.outcome-1] |
| 251 failure = xml_document.createElement(elem_name) |
| 252 testcase.appendChild(failure) |
| 253 |
| 254 failure.setAttribute('type', str(test_result.err[0].__name__)) |
| 255 failure.setAttribute('message', str(test_result.err[1])) |
| 256 |
| 257 error_info = test_result.get_error_info() |
| 258 failureText = xml_document.createCDATAOrText(error_info) |
| 259 failure.appendChild(failureText) |
| 260 |
| 261 _report_testcase = staticmethod(_report_testcase) |
| 262 |
| 263 def _report_output(test_runner, xml_testsuite, xml_document, stdout, stderr)
: |
| 264 "Appends the system-out and system-err sections to the XML document." |
| 265 systemout = xml_document.createElement('system-out') |
| 266 xml_testsuite.appendChild(systemout) |
| 267 |
| 268 systemout_text = xml_document.createCDATAOrText(stdout) |
| 269 systemout.appendChild(systemout_text) |
| 270 |
| 271 systemerr = xml_document.createElement('system-err') |
| 272 xml_testsuite.appendChild(systemerr) |
| 273 |
| 274 systemerr_text = xml_document.createCDATAOrText(stderr) |
| 275 systemerr.appendChild(systemerr_text) |
| 276 |
| 277 _report_output = staticmethod(_report_output) |
| 278 |
| 279 def generate_reports(self, test_runner): |
| 280 "Generates the XML reports to a given XMLTestRunner object." |
| 281 all_results = self._get_info_by_testcase() |
| 282 |
| 283 if type(test_runner.output) == str and not \ |
| 284 os.path.exists(test_runner.output): |
| 285 os.makedirs(test_runner.output) |
| 286 |
| 287 for suite, tests in all_results.items(): |
| 288 doc = XMLDocument() |
| 289 |
| 290 # Build the XML file |
| 291 testsuite = _XMLTestResult._report_testsuite(suite, tests, doc) |
| 292 stdout, stderr = [], [] |
| 293 for test in tests: |
| 294 _XMLTestResult._report_testcase(suite, test, testsuite, doc) |
| 295 if test.stdout: |
| 296 stdout.extend(['*****************', test.get_description(),
test.stdout]) |
| 297 if test.stderr: |
| 298 stderr.extend(['*****************', test.get_description(),
test.stderr]) |
| 299 _XMLTestResult._report_output(test_runner, testsuite, doc, |
| 300 '\n'.join(stdout), '\n'.join(stderr)) |
| 301 xml_content = doc.toprettyxml(indent='\t') |
| 302 |
| 303 if type(test_runner.output) is str: |
| 304 report_file = open('%s%sTEST-%s.xml' % \ |
| 305 (test_runner.output, os.sep, suite), 'w') |
| 306 try: |
| 307 report_file.write(xml_content) |
| 308 finally: |
| 309 report_file.close() |
| 310 else: |
| 311 # Assume that test_runner.output is a stream |
| 312 test_runner.output.write(xml_content) |
| 313 |
| 314 |
| 315 class XMLTestRunner(TextTestRunner): |
| 316 """A test runner class that outputs the results in JUnit like XML files. |
| 317 """ |
| 318 def __init__(self, output='.', stream=sys.stderr, descriptions=True, \ |
| 319 verbose=False, elapsed_times=True): |
| 320 "Create a new instance of XMLTestRunner." |
| 321 verbosity = (1, 2)[verbose] |
| 322 TextTestRunner.__init__(self, stream, descriptions, verbosity) |
| 323 self.output = output |
| 324 self.elapsed_times = elapsed_times |
| 325 |
| 326 def _make_result(self): |
| 327 """Create the TestResult object which will be used to store |
| 328 information about the executed tests. |
| 329 """ |
| 330 return _XMLTestResult(self.stream, self.descriptions, \ |
| 331 self.verbosity, self.elapsed_times) |
| 332 |
| 333 def run(self, test): |
| 334 "Run the given test case or test suite." |
| 335 # Prepare the test execution |
| 336 result = self._make_result() |
| 337 |
| 338 # Print a nice header |
| 339 self.stream.writeln() |
| 340 self.stream.writeln('Running tests...') |
| 341 self.stream.writeln(result.separator2) |
| 342 |
| 343 # Execute tests |
| 344 start_time = time.time() |
| 345 test(result) |
| 346 stop_time = time.time() |
| 347 time_taken = stop_time - start_time |
| 348 |
| 349 # Print results |
| 350 result.printErrors() |
| 351 self.stream.writeln(result.separator2) |
| 352 run = result.testsRun |
| 353 self.stream.writeln("Ran %d test%s in %.3fs" % |
| 354 (run, run != 1 and "s" or "", time_taken)) |
| 355 self.stream.writeln() |
| 356 |
| 357 # Error traces |
| 358 if not result.wasSuccessful(): |
| 359 self.stream.write("FAILED (") |
| 360 failed, errored = (len(result.failures), len(result.errors)) |
| 361 if failed: |
| 362 self.stream.write("failures=%d" % failed) |
| 363 if errored: |
| 364 if failed: |
| 365 self.stream.write(", ") |
| 366 self.stream.write("errors=%d" % errored) |
| 367 self.stream.writeln(")") |
| 368 else: |
| 369 self.stream.writeln("OK") |
| 370 |
| 371 # Generate reports |
| 372 self.stream.writeln() |
| 373 self.stream.writeln('Generating XML reports...') |
| 374 result.generate_reports(self) |
| 375 |
| 376 return result |
OLD | NEW |