Add image-first-tests flag to run-webkit-tests

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py

 # Copyright (C) 2011 Google Inc. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 # notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 # copyright notice, this list of conditions and the following disclaimer
@@ skipping 52 matching lines @@
         self._port = port
         self._filesystem = port.host.filesystem
         self._options = options
         self._results_directory = results_directory
         self._driver = primary_driver
         self._reference_driver = primary_driver
         self._timeout_ms = test_input.timeout_ms
         self._worker_name = worker_name
         self._test_name = test_input.test_name
         self._should_run_pixel_test = test_input.should_run_pixel_test
+        self._should_run_pixel_test_first = (
+            test_input.should_run_pixel_test_first)
         self._reference_files = test_input.reference_files
         self._should_add_missing_baselines = test_input.should_add_missing_baselines
         self._stop_when_done = stop_when_done
 
         # If this is a virtual test that uses the default flags instead of the
         # virtual flags for it's references, run it on the secondary driver so
         # that the primary driver does not need to be restarted.
         if (secondary_driver and
                 self._port.is_virtual_test(self._test_name) and
                 not self._port.lookup_virtual_reference_args(self._test_name)):
@@ skipping 193 matching lines @@
 
         if driver_output.crash:
             # Don't continue any more if we already have a crash.
             # In case of timeouts, we continue since we still want to see the text and image output.
             return TestResult(self._test_name, failures, driver_output.test_time, driver_output.has_stderr(),
                               pid=driver_output.pid)
 
         is_testharness_test, testharness_failures = self._compare_testharness_test(driver_output, expected_driver_output)
         if is_testharness_test:
             failures.extend(testharness_failures)
+
+        compare_functions = []
+        compare_image_fn = (self._compare_image, (
+            expected_driver_output, driver_output))
+        compare_txt_fn = (self._compare_text, (
+            expected_driver_output.text, driver_output.text))
+        compare_audio_fn = (self._compare_audio, (
+            expected_driver_output.audio, driver_output.audio))

[qyearsley, 2017/03/21 22:44:09]

Formatting note: Technically, the line length limit in Tools/Scripts/webkitpy is
132 so you could use longer lines if you want to.

+
+        if self._should_run_pixel_test_first:
+            if driver_output.image_hash:
+                compare_functions.append(compare_image_fn)
+            else:
+                compare_functions.append(compare_txt_fn)
         else:
-            failures.extend(self._compare_text(expected_driver_output.text, driver_output.text))
-        failures.extend(self._compare_audio(expected_driver_output.audio, driver_output.audio))
-        if self._should_run_pixel_test:
-            failures.extend(self._compare_image(expected_driver_output, driver_output))
+            compare_functions.append(compare_txt_fn)
+            if self._should_run_pixel_test:
+                compare_functions.append(compare_image_fn)
+        compare_functions.append(compare_audio_fn)
+
+        for func, args in compare_functions:
+            failures.extend(func(*args))
+
         has_repaint_overlay = (repaint_overlay.result_contains_repaint_rects(expected_driver_output.text) or
                                repaint_overlay.result_contains_repaint_rects(driver_output.text))
         return TestResult(self._test_name, failures, driver_output.test_time, driver_output.has_stderr(),
                           pid=driver_output.pid, has_repaint_overlay=has_repaint_overlay)
 
     def _compare_testharness_test(self, driver_output, expected_driver_output):
         if expected_driver_output.text:
             return False, []
 
         if self._is_render_tree(driver_output.text):
@@ skipping 148 matching lines @@
         elif reference_driver_output.image_hash != actual_driver_output.image_hash:
             diff, err_str = self._port.diff_image(reference_driver_output.image, actual_driver_output.image)
             if diff:
                 failures.append(test_failures.FailureReftestMismatch(reference_filename))
             elif err_str:
                 _log.error(err_str)
             else:
                 _log.warning("  %s -> ref test hashes didn't match but diff passed", self._test_name)
 
         return TestResult(self._test_name, failures, 0, has_stderr, pid=actual_driver_output.pid)