Allow text expectation for reftests

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py

 # Copyright (C) 2011 Google Inc. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 # notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 # copyright notice, this list of conditions and the following disclaimer
@@ skipping 67 matching lines @@
         # virtual flags for it's references, run it on the secondary driver so
         # that the primary driver does not need to be restarted.
         if (secondary_driver and
                 self._port.is_virtual_test(self._test_name) and
                 not self._port.lookup_virtual_reference_args(self._test_name)):
             self._reference_driver = secondary_driver
 
         if self._reference_files:
             # Detect and report a test which has a wrong combination of expectation files.
             # For example, if 'foo.html' has two expectation files, 'foo-expected.html' and
-            # 'foo-expected.txt', we should warn users. One test file must be used exclusively
-            # in either layout tests or reftests, but not in both.
-            for suffix in ('.txt', '.png', '.wav'):
+            # 'foo-expected.png', we should warn users. One test file must be used exclusively
+            # in either layout tests or reftests, but not in both. Text expectation is an
+            # exception.
+            for suffix in self._port.baseline_extensions():
+                if suffix == '.txt':
+                    continue
                 expected_filename = self._port.expected_filename(self._test_name, suffix)
                 if self._filesystem.exists(expected_filename):
                     _log.error('%s is a reftest, but has an unused expectation file. Please remove %s.',
                                self._test_name, expected_filename)
 
     def _expected_driver_output(self):
         return DriverOutput(self._port.expected_text(self._test_name),
                             self._port.expected_image(self._test_name),
                             self._port.expected_checksum(self._test_name),
                             self._port.expected_audio(self._test_name))
@@ skipping 20 matching lines @@
                 test_name = test_base
             args = self._port.lookup_virtual_test_args(self._test_name)
         else:
             test_name = self._test_name
             args = self._port.lookup_physical_test_args(self._test_name)
         return DriverInput(test_name, self._timeout, image_hash, self._should_run_pixel_test, args)
 
     def run(self):
         if self._options.enable_sanitizer:
             return self._run_sanitized_test()
+        if self._options.reset_results:
+            if self._reference_files:
+                expected_txt_filename = self._port.expected_filename(self._test_name, '.txt')
+                if not self._filesystem.exists(expected_txt_filename):
+                    reftest_type = set([reference_file[0] for reference_file in self._reference_files])
+                    result = TestResult(self._test_name, reftest_type=reftest_type)
+                    result.type = test_expectations.SKIP
+                    return result
+                self._should_run_pixel_test = False
+            return self._run_rebaseline()
         if self._reference_files:
-            if self._options.reset_results:
-                reftest_type = set([reference_file[0] for reference_file in self._reference_files])
-                result = TestResult(self._test_name, reftest_type=reftest_type)
-                result.type = test_expectations.SKIP
-                return result
             return self._run_reftest()
-        if self._options.reset_results:
-            return self._run_rebaseline()
         return self._run_compare_test()
 
     def _run_sanitized_test(self):
         # running a sanitized test means that we ignore the actual test output and just look
         # for timeouts and crashes (real or forced by the driver). Most crashes should
         # indicate problems found by a sanitizer (ASAN, LSAN, etc.), but we will report
         # on other crashes and timeouts as well in order to detect at least *some* basic failures.
         driver_output = self._driver.run_test(self._driver_input(), self._stop_when_done)
         expected_driver_output = self._expected_driver_output()
         failures = self._handle_error(driver_output)
@@ skipping 206 matching lines @@
                 if driver_output.image_diff:
                     failures.append(test_failures.FailureImageHashMismatch())
                 else:
                     # See https://bugs.webkit.org/show_bug.cgi?id=69444 for why this isn't a full failure.
                     _log.warning('  %s -> pixel hash failed (but diff passed)' % self._test_name)
         return failures
 
     def _run_reftest(self):
         test_output = self._driver.run_test(self._driver_input(), self._stop_when_done)
         total_test_time = test_output.test_time
-        reference_output = None
+        expected_output = None
         test_result = None
 
+        expected_text = self._port.expected_text(self._test_name)
+        expected_text_output = DriverOutput(text=expected_text, image=None, image_hash=None, audio=None)
+
         # If the test crashed, or timed out, there's no point in running the reference at all.
         # This can save a lot of execution time if we have a lot of crashes or timeouts.
         if test_output.crash or test_output.timeout:
-            expected_driver_output = DriverOutput(text=None, image=None, image_hash=None, audio=None)
-            test_result = self._compare_output(expected_driver_output, test_output)
+            test_result = self._compare_output(expected_text_output, test_output)
 
             if test_output.crash:
                 test_result_writer.write_test_result(self._filesystem, self._port, self._results_directory,
-                                                     self._test_name, test_output, expected_driver_output, test_result.failures)
+                                                     self._test_name, test_output, expected_text_output, test_result.failures)
             return test_result
 
         # A reftest can have multiple match references and multiple mismatch references;
         # the test fails if any mismatch matches and all of the matches don't match.
         # To minimize the number of references we have to check, we run all of the mismatches first,
         # then the matches, and short-circuit out as soon as we can.
         # Note that sorting by the expectation sorts "!=" before "==" so this is easy to do.
 
         putAllMismatchBeforeMatch = sorted
         reference_test_names = []
         for expectation, reference_filename in putAllMismatchBeforeMatch(self._reference_files):
             if self._port.lookup_virtual_test_base(self._test_name):
                 args = self._port.lookup_virtual_reference_args(self._test_name)
             else:
                 args = self._port.lookup_physical_reference_args(self._test_name)
             reference_test_name = self._port.relative_test_filename(reference_filename)
             reference_test_names.append(reference_test_name)
             driver_input = DriverInput(reference_test_name, self._timeout,
                                        image_hash=test_output.image_hash, should_run_pixel_test=True, args=args)
-            reference_output = self._reference_driver.run_test(driver_input, self._stop_when_done)
-            total_test_time += reference_output.test_time
+            expected_output = self._reference_driver.run_test(driver_input, self._stop_when_done)
+            total_test_time += expected_output.test_time
             test_result = self._compare_output_with_reference(
-                reference_output, test_output, reference_filename, expectation == '!=')
+                expected_output, test_output, reference_filename, expectation == '!=')
 
             if (expectation == '!=' and test_result.failures) or (expectation == '==' and not test_result.failures):
                 break
 
-        assert(reference_output)
+        assert(expected_output)
+
+        if expected_text:
+            text_output = DriverOutput(text=test_output.text, image=None, image_hash=None, audio=None)
+            test_result.failures.extend(self._compare_output(expected_text_output, text_output).failures)
+            expected_output.text = expected_text_output.text
+
         test_result_writer.write_test_result(self._filesystem, self._port, self._results_directory,
-                                             self._test_name, test_output, reference_output, test_result.failures)
+                                             self._test_name, test_output, expected_output, test_result.failures)
 
         # FIXME: We don't really deal with a mix of reftest types properly. We pass in a set() to reftest_type
         # and only really handle the first of the references in the result.
         reftest_type = list(set([reference_file[0] for reference_file in self._reference_files]))
         return TestResult(self._test_name, test_result.failures, total_test_time,
                           test_result.has_stderr, reftest_type=reftest_type, pid=test_result.pid,
                           references=reference_test_names)
 
     # The returned TestResult always has 0 test_run_time. _run_reftest() calculates total_run_time from test outputs.
     def _compare_output_with_reference(self, reference_driver_output, actual_driver_output, reference_filename, mismatch):
@@ skipping 15 matching lines @@
         elif reference_driver_output.image_hash != actual_driver_output.image_hash:
             diff, err_str = self._port.diff_image(reference_driver_output.image, actual_driver_output.image)
             if diff:
                 failures.append(test_failures.FailureReftestMismatch(reference_filename))
             elif err_str:
                 _log.error(err_str)
             else:
                 _log.warning("  %s -> ref test hashes didn't match but diff passed" % self._test_name)
 
         return TestResult(self._test_name, failures, 0, has_stderr, pid=actual_driver_output.pid)