Allow text expectation for reftests

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py

 # Copyright (C) 2010 Google Inc. All rights reserved.
 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
 # Copyright (C) 2011 Apple Inc. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 # notice, this list of conditions and the following disclaimer.
@@ skipping 26 matching lines @@
 import unittest
 
 from webkitpy.common.system import outputcapture, path
 from webkitpy.common.system.crashlogs_unittest import make_mock_crash_report_darwin
 from webkitpy.common.system.systemhost import SystemHost
 from webkitpy.common.host import Host
 from webkitpy.common.host_mock import MockHost
 
 from webkitpy.layout_tests import port
 from webkitpy.layout_tests import run_webkit_tests
+from webkitpy.layout_tests.models import test_expectations
+from webkitpy.layout_tests.models import test_failures
 from webkitpy.layout_tests.models import test_run_results
 from webkitpy.layout_tests.port import test
 from webkitpy.layout_tests.port.base import Port
 from webkitpy.tool import grammar
 from webkitpy.tool.mocktool import MockOptions
 
 
 def parse_args(extra_args=None, tests_included=False, new_results=False, print_nothing=True):
     extra_args = extra_args or []
     args = []
@@ skipping 761 matching lines @@
         test_name = 'virtual/references_use_default_args/passes/reftest.html'
         run_details, err, _ = logging_run(['--details', test_name], tests_included=True)
         self.assertEqual(run_details.exit_code, 0)
         self.assertEqual(run_details.initial_results.total, 1)
         test_result = run_details.initial_results.all_results[0]
         self.assertEqual(test_result.test_name, test_name)
         self.assertEqual(test_result.references[0], 'passes/reftest-expected.html')
         # reference_args should be empty since we are using the default flags.
         self.assertTrue(re.search('reference_args:\s*ref:', err.getvalue()))
 
+    def test_reftest_matching_text_expectation(self):
+        test_name = 'passes/reftest.html'
+        host = MockHost()
+        host.filesystem.write_text_file(test.LAYOUT_TEST_DIR + '/passes/reftest-expected.txt', 'reftest')
+        run_details, err, _ = logging_run([test_name], tests_included=True, host=host)
+        self.assertEqual(run_details.exit_code, 0)
+        self.assertEqual(run_details.initial_results.total, 1)
+        test_result = run_details.initial_results.all_results[0]
+        self.assertEqual(test_result.test_name, test_name)
+        self.assertEqual(len(test_result.failures), 0)
+
+    def test_reftest_mismatching_text_expectation(self):
+        test_name = 'passes/reftest.html'
+        host = MockHost()
+        host.filesystem.write_text_file(test.LAYOUT_TEST_DIR + '/passes/reftest-expected.txt', 'mismatch')
+        run_details, err, _ = logging_run([test_name], tests_included=True, host=host)
+        self.assertNotEqual(run_details.exit_code, 0)
+        self.assertEqual(run_details.initial_results.total, 1)
+        test_result = run_details.initial_results.all_results[0]
+        self.assertEqual(test_result.test_name, test_name)
+        self.assertEqual(len(test_result.failures), 1)
+        self.assertEqual(test_failures.determine_result_type(test_result.failures), test_expectations.TEXT)
+
+    def test_reftest_mismatching_pixel_matching_text(self):
+        test_name = 'failures/unexpected/reftest.html'
+        host = MockHost()
+        host.filesystem.write_text_file(test.LAYOUT_TEST_DIR + '/failures/unexpected/reftest-expected.txt', 'reftest')
+        run_details, err, _ = logging_run([test_name], tests_included=True, host=host)
+        self.assertNotEqual(run_details.exit_code, 0)
+        self.assertEqual(run_details.initial_results.total, 1)
+        test_result = run_details.initial_results.all_results[0]
+        self.assertEqual(test_result.test_name, test_name)
+        self.assertEqual(len(test_result.failures), 1)
+        self.assertEqual(test_failures.determine_result_type(test_result.failures), test_expectations.IMAGE)
+
+    def test_reftest_mismatching_both_text_and_pixel(self):
+        test_name = 'failures/unexpected/reftest.html'
+        host = MockHost()
+        host.filesystem.write_text_file(test.LAYOUT_TEST_DIR + '/failures/unexpected/reftest-expected.txt', 'mismatch')
+        run_details, err, _ = logging_run([test_name], tests_included=True, host=host)
+        self.assertNotEqual(run_details.exit_code, 0)
+        self.assertEqual(run_details.initial_results.total, 1)
+        test_result = run_details.initial_results.all_results[0]
+        self.assertEqual(test_result.test_name, test_name)
+        self.assertEqual(len(test_result.failures), 2)
+        self.assertEqual(test_failures.determine_result_type(test_result.failures), test_expectations.IMAGE_PLUS_TEXT)
+
     def test_additional_platform_directory(self):
         self.assertTrue(passing_run(['--additional-platform-directory', '/tmp/foo']))
         self.assertTrue(passing_run(['--additional-platform-directory', '/tmp/../foo']))
         self.assertTrue(passing_run(['--additional-platform-directory', '/tmp/foo', '--additional-platform-directory', '/tmp/bar']))
         self.assertTrue(passing_run(['--additional-platform-directory', 'foo']))
 
     def test_additional_expectations(self):
         host = MockHost()
         host.filesystem.write_text_file('/tmp/overrides.txt', 'Bug(x) failures/unexpected/mismatch.html [ Failure ]\n')
         self.assertTrue(passing_run(['--additional-expectations', '/tmp/overrides.txt', 'failures/unexpected/mismatch.html'],
@@ skipping 222 matching lines @@
             ['--pixel-tests', '--new-baseline', 'passes/image.html', 'failures/expected/missing_image.html'],
             tests_included=True, host=host, new_results=True)
         file_list = host.filesystem.written_files.keys()
         self.assertEqual(details.exit_code, 0)
         self.assertEqual(len(file_list), 9)
         self.assertBaselines(file_list,
                              "platform/test-mac-mac10.10/passes/image", [".txt", ".png"], err)
         self.assertBaselines(file_list,
                              "platform/test-mac-mac10.10/failures/expected/missing_image", [".txt", ".png"], err)
 
+    def test_reftest_reset_results(self):
+        # Test rebaseline of reftests.
+        # Should ignore reftests without text expectations.
+        host = MockHost()
+        details, err, _ = logging_run(['--reset-results', 'passes/reftest.html'], tests_included=True, host=host)
+        file_list = host.filesystem.written_files.keys()
+        self.assertEqual(details.exit_code, 0)
+        self.assertEqual(len(file_list), 5)
+        self.assertBaselines(file_list, '', [], err)
+
+        host.filesystem.write_text_file(test.LAYOUT_TEST_DIR + '/passes/reftest-expected.txt', '')
+        host.filesystem.clear_written_files()
+        details, err, _ = logging_run(['--reset-results', 'passes/reftest.html'], tests_included=True, host=host)
+        file_list = host.filesystem.written_files.keys()
+        self.assertEqual(details.exit_code, 0)
+        self.assertEqual(len(file_list), 6)
+        self.assertBaselines(file_list, 'passes/reftest', ['.txt'], err)
+
+    def test_reftest_new_baseline(self):
+        # Test rebaseline of reftests.
+        # Should ignore reftests without text expectations.
+        host = MockHost()
+        details, err, _ = logging_run(['--new-baseline', 'passes/reftest.html'], tests_included=True, host=host)
+        file_list = host.filesystem.written_files.keys()
+        self.assertEqual(details.exit_code, 0)
+        self.assertEqual(len(file_list), 5)
+        self.assertBaselines(file_list, '', [], err)
+
+        host.filesystem.write_text_file(test.LAYOUT_TEST_DIR + '/passes/reftest-expected.txt', '')
+        host.filesystem.clear_written_files()
+        details, err, _ = logging_run(['--new-baseline', 'passes/reftest.html'], tests_included=True, host=host)
+        file_list = host.filesystem.written_files.keys()
+        self.assertEqual(details.exit_code, 0)
+        self.assertEqual(len(file_list), 6)
+        self.assertBaselines(file_list, 'platform/test-mac-mac10.10/passes/reftest', ['.txt'], err)
+
 
 class PortTest(unittest.TestCase):
 
     def assert_mock_port_works(self, port_name, args=[]):
         self.assertTrue(passing_run(args + ['--platform', 'mock-' + port_name,
                                             'fast/harness/results.html'], tests_included=True, host=Host()))
 
     def disabled_test_mac_lion(self):
         self.assert_mock_port_works('mac-lion')
 
@@ skipping 26 matching lines @@
 
             run_webkit_tests._run_tests = successful_run
             res = run_webkit_tests.main(['--platform', 'test'], stdout, stderr)
             self.assertEqual(res, test_run_results.UNEXPECTED_ERROR_EXIT_STATUS)
 
             run_webkit_tests._run_tests = exception_raising_run
             res = run_webkit_tests.main([], stdout, stderr)
             self.assertEqual(res, test_run_results.UNEXPECTED_ERROR_EXIT_STATUS)
         finally:
             run_webkit_tests._run_tests = orig_run_fn