Index: third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py |
diff --git a/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py b/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py |
index 9be70ec5698bd5db165b1b68220b1c62e762bada..fd1d7c4a3badece10d7f260125db5e9461a47af1 100644 |
--- a/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py |
+++ b/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py |
@@ -172,6 +172,7 @@ def parse_full_results(full_results_text): |
class StreamTestingMixin(object): |
+ |
def assertContains(self, stream, string): |
self.assertTrue(string in stream.getvalue()) |
@@ -183,6 +184,7 @@ class StreamTestingMixin(object): |
class RunTest(unittest.TestCase, StreamTestingMixin): |
+ |
def setUp(self): |
# A real PlatformInfo object is used here instead of a |
# MockPlatformInfo because we need to actually check for |
@@ -208,10 +210,12 @@ class RunTest(unittest.TestCase, StreamTestingMixin): |
self.assertEqual(details.exit_code, test.UNEXPECTED_PASSES) |
self.assertEqual(details.all_retry_results[0].total, test.UNEXPECTED_PASSES) |
- expected_tests = details.initial_results.total - details.initial_results.expected_skips - len(details.initial_results.unexpected_results_by_name) - test.TOTAL_CRASHES |
+ expected_tests = details.initial_results.total - details.initial_results.expected_skips - \ |
+ len(details.initial_results.unexpected_results_by_name) - test.TOTAL_CRASHES |
expected_summary_str = '' |
if details.initial_results.expected_failures > 0: |
- expected_summary_str = " (%d passed, %d didn't)" % (expected_tests - details.initial_results.expected_failures, details.initial_results.expected_failures) |
+ expected_summary_str = " (%d passed, %d didn't)" % ( |
+ expected_tests - details.initial_results.expected_failures, details.initial_results.expected_failures) |
one_line_summary = "%d tests ran as expected%s, %d didn't (%d didn't run):\n" % ( |
expected_tests, |
expected_summary_str, |
@@ -286,11 +290,11 @@ class RunTest(unittest.TestCase, StreamTestingMixin): |
# WorkerExceptions (a subclass of BaseException), which have a string capture of the stack which can |
# be printed, but don't display properly in the unit test exception handlers. |
self.assertRaises(BaseException, logging_run, |
- ['failures/expected/exception.html', '--child-processes', '1'], tests_included=True) |
+ ['failures/expected/exception.html', '--child-processes', '1'], tests_included=True) |
if self.should_test_processes: |
self.assertRaises(BaseException, logging_run, |
- ['--child-processes', '2', '--skipped=ignore', 'failures/expected/exception.html', 'passes/text.html'], tests_included=True, shared_port=False) |
+ ['--child-processes', '2', '--skipped=ignore', 'failures/expected/exception.html', 'passes/text.html'], tests_included=True, shared_port=False) |
def test_device_failure(self): |
# Test that we handle a device going offline during a test properly. |
@@ -311,7 +315,8 @@ class RunTest(unittest.TestCase, StreamTestingMixin): |
self.assertEqual(details.exit_code, test_run_results.INTERRUPTED_EXIT_STATUS) |
if self.should_test_processes: |
- _, regular_output, _ = logging_run(['failures/expected/keyboard.html', 'passes/text.html', '--child-processes', '2', '--skipped=ignore'], tests_included=True, shared_port=False) |
+ _, regular_output, _ = logging_run(['failures/expected/keyboard.html', 'passes/text.html', |
+ '--child-processes', '2', '--skipped=ignore'], tests_included=True, shared_port=False) |
self.assertTrue(any(['Interrupted, exiting' in line for line in regular_output.buflist])) |
def test_no_tests_found(self): |
@@ -330,9 +335,11 @@ class RunTest(unittest.TestCase, StreamTestingMixin): |
self.assertContains(err, 'No tests to run.\n') |
def test_natural_order(self): |
- tests_to_run = ['passes/audio.html', 'failures/expected/text.html', 'failures/expected/missing_text.html', 'passes/args.html'] |
+ tests_to_run = ['passes/audio.html', 'failures/expected/text.html', |
+ 'failures/expected/missing_text.html', 'passes/args.html'] |
tests_run = get_tests_run(['--order=natural'] + tests_to_run) |
- self.assertEqual(['failures/expected/missing_text.html', 'failures/expected/text.html', 'passes/args.html', 'passes/audio.html'], tests_run) |
+ self.assertEqual(['failures/expected/missing_text.html', 'failures/expected/text.html', |
+ 'passes/args.html', 'passes/audio.html'], tests_run) |
def test_natural_order_test_specified_multiple_times(self): |
tests_to_run = ['passes/args.html', 'passes/audio.html', 'passes/audio.html', 'passes/args.html'] |
@@ -340,12 +347,14 @@ class RunTest(unittest.TestCase, StreamTestingMixin): |
self.assertEqual(['passes/args.html', 'passes/args.html', 'passes/audio.html', 'passes/audio.html'], tests_run) |
def test_random_order(self): |
- tests_to_run = ['passes/audio.html', 'failures/expected/text.html', 'failures/expected/missing_text.html', 'passes/args.html'] |
+ tests_to_run = ['passes/audio.html', 'failures/expected/text.html', |
+ 'failures/expected/missing_text.html', 'passes/args.html'] |
tests_run = get_tests_run(['--order=random'] + tests_to_run) |
self.assertEqual(sorted(tests_to_run), sorted(tests_run)) |
def test_random_daily_seed_order(self): |
- tests_to_run = ['passes/audio.html', 'failures/expected/text.html', 'failures/expected/missing_text.html', 'passes/args.html'] |
+ tests_to_run = ['passes/audio.html', 'failures/expected/text.html', |
+ 'failures/expected/missing_text.html', 'passes/args.html'] |
tests_run = get_tests_run(['--order=random-seeded'] + tests_to_run) |
self.assertEqual(sorted(tests_to_run), sorted(tests_run)) |
@@ -356,7 +365,8 @@ class RunTest(unittest.TestCase, StreamTestingMixin): |
self.assertEqual(tests_run.count('passes/args.html'), 2) |
def test_no_order(self): |
- tests_to_run = ['passes/audio.html', 'failures/expected/text.html', 'failures/expected/missing_text.html', 'passes/args.html'] |
+ tests_to_run = ['passes/audio.html', 'failures/expected/text.html', |
+ 'failures/expected/missing_text.html', 'passes/args.html'] |
tests_run = get_tests_run(['--order=none'] + tests_to_run) |
self.assertEqual(tests_to_run, tests_run) |
@@ -368,7 +378,8 @@ class RunTest(unittest.TestCase, StreamTestingMixin): |
def test_no_order_with_directory_entries_in_natural_order(self): |
tests_to_run = ['http/tests/ssl', 'perf/foo', 'http/tests/passes'] |
tests_run = get_tests_run(['--order=none'] + tests_to_run) |
- self.assertEqual(tests_run, ['http/tests/ssl/text.html', 'perf/foo/test.html', 'http/tests/passes/image.html', 'http/tests/passes/text.html']) |
+ self.assertEqual(tests_run, ['http/tests/ssl/text.html', 'perf/foo/test.html', |
+ 'http/tests/passes/image.html', 'http/tests/passes/text.html']) |
def test_repeat_each(self): |
tests_to_run = ['passes/image.html', 'passes/text.html'] |
@@ -388,7 +399,7 @@ class RunTest(unittest.TestCase, StreamTestingMixin): |
# Check that nothing changes when we specify skipped=default. |
self.assertEqual(len(get_tests_run(['--skipped=default', 'passes'])), |
- num_tests_run_by_default) |
+ num_tests_run_by_default) |
# Now check that we run one more test (the skipped one). |
tests_run = get_tests_run(['--skipped=ignore', 'passes']) |
@@ -524,15 +535,17 @@ class RunTest(unittest.TestCase, StreamTestingMixin): |
# is missing, update the expected generic location. |
host = MockHost() |
details, err, _ = logging_run(['--no-show-results', '--retry-failures', |
- 'failures/expected/missing_image.html', |
- 'failures/unexpected/missing_text.html', |
- 'failures/unexpected/text-image-checksum.html'], |
- tests_included=True, host=host) |
+ 'failures/expected/missing_image.html', |
+ 'failures/unexpected/missing_text.html', |
+ 'failures/unexpected/text-image-checksum.html'], |
+ tests_included=True, host=host) |
file_list = host.filesystem.written_files.keys() |
self.assertEqual(details.exit_code, 2) |
json_string = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json') |
- self.assertTrue(json_string.find('"text-image-checksum.html":{"expected":"PASS","actual":"IMAGE+TEXT","is_unexpected":true') != -1) |
- self.assertTrue(json_string.find('"missing_text.html":{"expected":"PASS","is_missing_text":true,"actual":"MISSING","is_unexpected":true') != -1) |
+ self.assertTrue(json_string.find( |
+ '"text-image-checksum.html":{"expected":"PASS","actual":"IMAGE+TEXT","is_unexpected":true') != -1) |
+ self.assertTrue(json_string.find( |
+ '"missing_text.html":{"expected":"PASS","is_missing_text":true,"actual":"MISSING","is_unexpected":true') != -1) |
self.assertTrue(json_string.find('"num_regressions":2') != -1) |
self.assertTrue(json_string.find('"num_flaky":0') != -1) |
@@ -572,18 +585,21 @@ class RunTest(unittest.TestCase, StreamTestingMixin): |
def test_crash_with_stderr(self): |
host = MockHost() |
_, regular_output, _ = logging_run(['failures/unexpected/crash-with-stderr.html'], tests_included=True, host=host) |
- self.assertTrue(host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json').find('{"crash-with-stderr.html":{"expected":"PASS","actual":"CRASH","has_stderr":true,"is_unexpected":true') != -1) |
+ self.assertTrue(host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json').find( |
+ '{"crash-with-stderr.html":{"expected":"PASS","actual":"CRASH","has_stderr":true,"is_unexpected":true') != -1) |
def test_no_image_failure_with_image_diff(self): |
host = MockHost() |
- _, regular_output, _ = logging_run(['failures/unexpected/checksum-with-matching-image.html'], tests_included=True, host=host) |
- self.assertTrue(host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json').find('"num_regressions":0') != -1) |
+ _, regular_output, _ = logging_run( |
+ ['failures/unexpected/checksum-with-matching-image.html'], tests_included=True, host=host) |
+ self.assertTrue(host.filesystem.read_text_file( |
+ '/tmp/layout-test-results/full_results.json').find('"num_regressions":0') != -1) |
def test_exit_after_n_failures_upload(self): |
host = MockHost() |
details, regular_output, user = logging_run( |
- ['failures/unexpected/text-image-checksum.html', 'passes/text.html', '--exit-after-n-failures', '1'], |
- tests_included=True, host=host) |
+ ['failures/unexpected/text-image-checksum.html', 'passes/text.html', '--exit-after-n-failures', '1'], |
+ tests_included=True, host=host) |
# By returning False, we know that the incremental results were generated and then deleted. |
self.assertFalse(host.filesystem.exists('/tmp/layout-test-results/incremental_results.json')) |
@@ -602,7 +618,8 @@ class RunTest(unittest.TestCase, StreamTestingMixin): |
def test_exit_after_n_failures(self): |
# Unexpected failures should result in tests stopping. |
- tests_run = get_tests_run(['failures/unexpected/text-image-checksum.html', 'passes/text.html', '--exit-after-n-failures', '1']) |
+ tests_run = get_tests_run(['failures/unexpected/text-image-checksum.html', |
+ 'passes/text.html', '--exit-after-n-failures', '1']) |
self.assertEqual(['failures/unexpected/text-image-checksum.html'], tests_run) |
# But we'll keep going for expected ones. |
@@ -615,7 +632,8 @@ class RunTest(unittest.TestCase, StreamTestingMixin): |
self.assertEqual(['failures/unexpected/crash.html'], tests_run) |
# Same with timeouts. |
- tests_run = get_tests_run(['failures/unexpected/timeout.html', 'passes/text.html', '--exit-after-n-crashes-or-timeouts', '1']) |
+ tests_run = get_tests_run(['failures/unexpected/timeout.html', 'passes/text.html', |
+ '--exit-after-n-crashes-or-timeouts', '1']) |
self.assertEqual(['failures/unexpected/timeout.html'], tests_run) |
# But we'll keep going for expected ones. |
@@ -650,7 +668,8 @@ class RunTest(unittest.TestCase, StreamTestingMixin): |
def test_retrying_default_value(self): |
host = MockHost() |
- details, err, _ = logging_run(['--debug-rwt-logging', 'failures/unexpected/text-image-checksum.html'], tests_included=True, host=host) |
+ details, err, _ = logging_run( |
+ ['--debug-rwt-logging', 'failures/unexpected/text-image-checksum.html'], tests_included=True, host=host) |
self.assertEqual(details.exit_code, 1) |
self.assertFalse('Retrying' in err.getvalue()) |
@@ -688,7 +707,8 @@ class RunTest(unittest.TestCase, StreamTestingMixin): |
# Now we test that --clobber-old-results does remove the old entries and the old retries, |
# and that we don't retry again. |
host = MockHost() |
- details, err, _ = logging_run(['--no-retry-failures', '--clobber-old-results', 'failures/flaky'], tests_included=True, host=host) |
+ details, err, _ = logging_run(['--no-retry-failures', '--clobber-old-results', |
+ 'failures/flaky'], tests_included=True, host=host) |
self.assertEqual(details.exit_code, 1) |
self.assertTrue('Clobbering old results' in err.getvalue()) |
self.assertTrue('flaky/text.html' in err.getvalue()) |
@@ -712,7 +732,8 @@ class RunTest(unittest.TestCase, StreamTestingMixin): |
def test_retrying_force_pixel_tests(self): |
host = MockHost() |
- details, err, _ = logging_run(['--no-pixel-tests', '--retry-failures', 'failures/unexpected/text-image-checksum.html'], tests_included=True, host=host) |
+ details, err, _ = logging_run(['--no-pixel-tests', '--retry-failures', |
+ 'failures/unexpected/text-image-checksum.html'], tests_included=True, host=host) |
self.assertEqual(details.exit_code, 1) |
self.assertTrue('Retrying' in err.getvalue()) |
self.assertTrue(host.filesystem.exists('/tmp/layout-test-results/failures/unexpected/text-image-checksum-actual.txt')) |
@@ -732,7 +753,8 @@ class RunTest(unittest.TestCase, StreamTestingMixin): |
def test_retrying_uses_retry_directories(self): |
host = MockHost() |
- details, err, _ = logging_run(['--debug-rwt-logging', '--retry-failures', 'failures/unexpected/text-image-checksum.html'], tests_included=True, host=host) |
+ details, err, _ = logging_run(['--debug-rwt-logging', '--retry-failures', |
+ 'failures/unexpected/text-image-checksum.html'], tests_included=True, host=host) |
self.assertEqual(details.exit_code, 1) |
self.assertTrue(host.filesystem.exists('/tmp/layout-test-results/failures/unexpected/text-image-checksum-actual.txt')) |
self.assertTrue(host.filesystem.exists('/tmp/layout-test-results/retry_1/failures/unexpected/text-image-checksum-actual.txt')) |
@@ -862,7 +884,8 @@ class RunTest(unittest.TestCase, StreamTestingMixin): |
port_name = 'mac-mac10.11' |
out = StringIO.StringIO() |
err = StringIO.StringIO() |
- self.assertEqual(run_webkit_tests.main(['--platform', port_name, 'fast/harness/results.html'], out, err), test_run_results.UNEXPECTED_ERROR_EXIT_STATUS) |
+ self.assertEqual(run_webkit_tests.main( |
+ ['--platform', port_name, 'fast/harness/results.html'], out, err), test_run_results.UNEXPECTED_ERROR_EXIT_STATUS) |
def test_verbose_in_child_processes(self): |
# When we actually run multiple processes, we may have to reconfigure logging in the |
@@ -874,7 +897,8 @@ class RunTest(unittest.TestCase, StreamTestingMixin): |
if not self.should_test_processes: |
return |
- options, parsed_args = parse_args(['--verbose', '--fully-parallel', '--child-processes', '2', 'passes/text.html', 'passes/image.html'], tests_included=True, print_nothing=False) |
+ options, parsed_args = parse_args(['--verbose', '--fully-parallel', '--child-processes', |
+ '2', 'passes/text.html', 'passes/image.html'], tests_included=True, print_nothing=False) |
host = MockHost() |
port_obj = host.port_factory.get(port_name=options.platform, options=options) |
logging_stream = StringIO.StringIO() |
@@ -913,6 +937,7 @@ class RunTest(unittest.TestCase, StreamTestingMixin): |
class EndToEndTest(unittest.TestCase): |
+ |
def test_reftest_with_two_notrefs(self): |
# Test that we update expectations in place. If the expectation |
# is missing, update the expected generic location. |
@@ -927,14 +952,15 @@ class EndToEndTest(unittest.TestCase): |
self.assertTrue("multiple-both-success.html" not in json["tests"]["reftests"]["foo"]) |
self.assertEqual(json["tests"]["reftests"]["foo"]["multiple-match-failure.html"], |
- {"expected": "PASS", "actual": "IMAGE", "reftest_type": ["=="], "is_unexpected": True}) |
+ {"expected": "PASS", "actual": "IMAGE", "reftest_type": ["=="], "is_unexpected": True}) |
self.assertEqual(json["tests"]["reftests"]["foo"]["multiple-mismatch-failure.html"], |
- {"expected": "PASS", "actual": "IMAGE", "reftest_type": ["!="], "is_unexpected": True}) |
+ {"expected": "PASS", "actual": "IMAGE", "reftest_type": ["!="], "is_unexpected": True}) |
self.assertEqual(json["tests"]["reftests"]["foo"]["multiple-both-failure.html"], |
- {"expected": "PASS", "actual": "IMAGE", "reftest_type": ["==", "!="], "is_unexpected": True}) |
+ {"expected": "PASS", "actual": "IMAGE", "reftest_type": ["==", "!="], "is_unexpected": True}) |
class RebaselineTest(unittest.TestCase, StreamTestingMixin): |
+ |
def assertBaselines(self, file_list, file, extensions, err): |
"assert that the file_list contains the baselines.""" |
for ext in extensions: |
@@ -964,10 +990,10 @@ class RebaselineTest(unittest.TestCase, StreamTestingMixin): |
# is missing, update the expected generic location. |
host = MockHost() |
details, err, _ = logging_run(['--no-show-results', |
- 'failures/unexpected/missing_text.html', |
- 'failures/unexpected/missing_image.html', |
- 'failures/unexpected/missing_render_tree_dump.html'], |
- tests_included=True, host=host, new_results=True) |
+ 'failures/unexpected/missing_text.html', |
+ 'failures/unexpected/missing_image.html', |
+ 'failures/unexpected/missing_render_tree_dump.html'], |
+ tests_included=True, host=host, new_results=True) |
file_list = host.filesystem.written_files.keys() |
self.assertEqual(details.exit_code, 3) |
self.assertEqual(len(file_list), 11) |
@@ -989,11 +1015,11 @@ Bug(foo) failures/unexpected/missing_audio.html [ NeedsManualRebaseline ] |
Bug(foo) failures/unexpected/missing_render_tree_dump.html [ Missing ] |
""") |
details, err, _ = logging_run(['--no-show-results', |
- 'failures/unexpected/missing_text.html', |
- 'failures/unexpected/missing_image.html', |
- 'failures/unexpected/missing_audio.html', |
- 'failures/unexpected/missing_render_tree_dump.html'], |
- tests_included=True, host=host, new_results=True, port_obj=port) |
+ 'failures/unexpected/missing_text.html', |
+ 'failures/unexpected/missing_image.html', |
+ 'failures/unexpected/missing_audio.html', |
+ 'failures/unexpected/missing_render_tree_dump.html'], |
+ tests_included=True, host=host, new_results=True, port_obj=port) |
file_list = host.filesystem.written_files.keys() |
self.assertEqual(details.exit_code, 0) |
self.assertEqual(len(file_list), 8) |
@@ -1015,11 +1041,11 @@ Bug(foo) failures/unexpected/missing_audio.html [ NeedsManualRebaseline ] |
Bug(foo) failures/unexpected/missing_render_tree_dump.html [ Missing ] |
""") |
details, err, _ = logging_run(['--pixel-tests', '--reset-results', |
- 'failures/unexpected/missing_text.html', |
- 'failures/unexpected/missing_image.html', |
- 'failures/unexpected/missing_audio.html', |
- 'failures/unexpected/missing_render_tree_dump.html'], |
- tests_included=True, host=host, new_results=True, port_obj=port) |
+ 'failures/unexpected/missing_text.html', |
+ 'failures/unexpected/missing_image.html', |
+ 'failures/unexpected/missing_audio.html', |
+ 'failures/unexpected/missing_render_tree_dump.html'], |
+ tests_included=True, host=host, new_results=True, port_obj=port) |
file_list = host.filesystem.written_files.keys() |
self.assertEqual(details.exit_code, 0) |
self.assertEqual(len(file_list), 12) |
@@ -1038,20 +1064,23 @@ Bug(foo) failures/unexpected/missing_render_tree_dump.html [ Missing ] |
self.assertEqual(details.exit_code, 0) |
self.assertEqual(len(file_list), 9) |
self.assertBaselines(file_list, |
- "platform/test-mac-mac10.10/passes/image", [".txt", ".png"], err) |
+ "platform/test-mac-mac10.10/passes/image", [".txt", ".png"], err) |
self.assertBaselines(file_list, |
- "platform/test-mac-mac10.10/failures/expected/missing_image", [".txt", ".png"], err) |
+ "platform/test-mac-mac10.10/failures/expected/missing_image", [".txt", ".png"], err) |
class PortTest(unittest.TestCase): |
+ |
def assert_mock_port_works(self, port_name, args=[]): |
- self.assertTrue(passing_run(args + ['--platform', 'mock-' + port_name, 'fast/harness/results.html'], tests_included=True, host=Host())) |
+ self.assertTrue(passing_run(args + ['--platform', 'mock-' + port_name, |
+ 'fast/harness/results.html'], tests_included=True, host=Host())) |
def disabled_test_mac_lion(self): |
self.assert_mock_port_works('mac-lion') |
class MainTest(unittest.TestCase): |
+ |
def test_exception_handling(self): |
orig_run_fn = run_webkit_tests._run_tests |