Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(1097)

Unified Diff: third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py

Issue 1346673003: Allow text expectation for reftests (Closed) Base URL: svn://svn.chromium.org/blink/trunk
Patch Set: Created 4 years, 7 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
Index: third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py
diff --git a/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py b/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py
index c7a8ca723410f0a7102720d4bce395519dbc7b07..055e69013573edbe0a409e35c15d5908ccba5d04 100644
--- a/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py
+++ b/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py
@@ -44,6 +44,8 @@ from webkitpy.common.host_mock import MockHost
from webkitpy.layout_tests import port
from webkitpy.layout_tests import run_webkit_tests
+from webkitpy.layout_tests.models import test_expectations
+from webkitpy.layout_tests.models import test_failures
from webkitpy.layout_tests.models import test_run_results
from webkitpy.layout_tests.port import test
from webkitpy.layout_tests.port.base import Port
@@ -825,6 +827,53 @@ class RunTest(unittest.TestCase, StreamTestingMixin):
# reference_args should be empty since we are using the default flags.
self.assertTrue(re.search('reference_args:\s*ref:', err.getvalue()))
+ def test_reftest_matching_text_expectation(self):
+ test_name = 'passes/reftest.html'
+ host = MockHost()
+ host.filesystem.write_text_file(test.LAYOUT_TEST_DIR + '/passes/reftest-expected.txt', 'reftest')
+ run_details, err, _ = logging_run([test_name], tests_included=True, host=host)
+ self.assertEqual(run_details.exit_code, 0)
+ self.assertEqual(run_details.initial_results.total, 1)
+ test_result = run_details.initial_results.all_results[0]
+ self.assertEqual(test_result.test_name, test_name)
+ self.assertEqual(len(test_result.failures), 0)
+
+ def test_reftest_mismatching_text_expectation(self):
+ test_name = 'passes/reftest.html'
+ host = MockHost()
+ host.filesystem.write_text_file(test.LAYOUT_TEST_DIR + '/passes/reftest-expected.txt', 'mismatch')
+ run_details, err, _ = logging_run([test_name], tests_included=True, host=host)
+ self.assertNotEqual(run_details.exit_code, 0)
+ self.assertEqual(run_details.initial_results.total, 1)
+ test_result = run_details.initial_results.all_results[0]
+ self.assertEqual(test_result.test_name, test_name)
+ self.assertEqual(len(test_result.failures), 1)
+ self.assertEqual(test_failures.determine_result_type(test_result.failures), test_expectations.TEXT)
+
+ def test_reftest_mismatching_pixel_matching_text(self):
+ test_name = 'failures/unexpected/reftest.html'
+ host = MockHost()
+ host.filesystem.write_text_file(test.LAYOUT_TEST_DIR + '/failures/unexpected/reftest-expected.txt', 'reftest')
+ run_details, err, _ = logging_run([test_name], tests_included=True, host=host)
+ self.assertNotEqual(run_details.exit_code, 0)
+ self.assertEqual(run_details.initial_results.total, 1)
+ test_result = run_details.initial_results.all_results[0]
+ self.assertEqual(test_result.test_name, test_name)
+ self.assertEqual(len(test_result.failures), 1)
+ self.assertEqual(test_failures.determine_result_type(test_result.failures), test_expectations.IMAGE)
+
+ def test_reftest_mismatching_both_text_and_pixel(self):
+ test_name = 'failures/unexpected/reftest.html'
+ host = MockHost()
+ host.filesystem.write_text_file(test.LAYOUT_TEST_DIR + '/failures/unexpected/reftest-expected.txt', 'mismatch')
+ run_details, err, _ = logging_run([test_name], tests_included=True, host=host)
+ self.assertNotEqual(run_details.exit_code, 0)
+ self.assertEqual(run_details.initial_results.total, 1)
+ test_result = run_details.initial_results.all_results[0]
+ self.assertEqual(test_result.test_name, test_name)
+ self.assertEqual(len(test_result.failures), 2)
+ self.assertEqual(test_failures.determine_result_type(test_result.failures), test_expectations.IMAGE_PLUS_TEXT)
+
def test_additional_platform_directory(self):
self.assertTrue(passing_run(['--additional-platform-directory', '/tmp/foo']))
self.assertTrue(passing_run(['--additional-platform-directory', '/tmp/../foo']))
@@ -1067,6 +1116,42 @@ Bug(foo) failures/unexpected/missing_render_tree_dump.html [ Missing ]
self.assertBaselines(file_list,
"platform/test-mac-mac10.10/failures/expected/missing_image", [".txt", ".png"], err)
+ def test_reftest_reset_results(self):
+ # Test rebaseline of reftests.
+ # Should ignore reftests without text expectations.
+ host = MockHost()
+ details, err, _ = logging_run(['--reset-results', 'passes/reftest.html'], tests_included=True, host=host)
+ file_list = host.filesystem.written_files.keys()
+ self.assertEqual(details.exit_code, 0)
+ self.assertEqual(len(file_list), 5)
+ self.assertBaselines(file_list, '', [], err)
+
+ host.filesystem.write_text_file(test.LAYOUT_TEST_DIR + '/passes/reftest-expected.txt', '')
+ host.filesystem.clear_written_files()
+ details, err, _ = logging_run(['--reset-results', 'passes/reftest.html'], tests_included=True, host=host)
+ file_list = host.filesystem.written_files.keys()
+ self.assertEqual(details.exit_code, 0)
+ self.assertEqual(len(file_list), 6)
+ self.assertBaselines(file_list, 'passes/reftest', ['.txt'], err)
+
+ def test_reftest_new_baseline(self):
+ # Test rebaseline of reftests.
+ # Should ignore reftests without text expectations.
+ host = MockHost()
+ details, err, _ = logging_run(['--new-baseline', 'passes/reftest.html'], tests_included=True, host=host)
+ file_list = host.filesystem.written_files.keys()
+ self.assertEqual(details.exit_code, 0)
+ self.assertEqual(len(file_list), 5)
+ self.assertBaselines(file_list, '', [], err)
+
+ host.filesystem.write_text_file(test.LAYOUT_TEST_DIR + '/passes/reftest-expected.txt', '')
+ host.filesystem.clear_written_files()
+ details, err, _ = logging_run(['--new-baseline', 'passes/reftest.html'], tests_included=True, host=host)
+ file_list = host.filesystem.written_files.keys()
+ self.assertEqual(details.exit_code, 0)
+ self.assertEqual(len(file_list), 6)
+ self.assertBaselines(file_list, 'platform/test-mac-mac10.10/passes/reftest', ['.txt'], err)
+
class PortTest(unittest.TestCase):

Powered by Google App Engine
This is Rietveld 408576698