Improve run-webkit-tests error reporting when ref tests have missing images.

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/models/test_failures.py

 # Copyright (C) 2010 Google Inc. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 # notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 # copyright notice, this list of conditions and the following disclaimer
@@ skipping 15 matching lines @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import cPickle
 
 from webkitpy.layout_tests.models import test_expectations
 
 
 def is_reftest_failure(failure_list):
     failure_types = [type(f) for f in failure_list]
-    return set((FailureReftestMismatch, FailureReftestMismatchDidNotOccur, FailureReftestNoImagesGenerated)).intersection(
-        failure_types)
+    return set((
+        FailureReftestMismatch,
+        FailureReftestMismatchDidNotOccur,
+        FailureReftestNoImageGenerated,
+        FailureReftestNoReferenceImageGenerated
+    )).intersection(failure_types)

[qyearsley, 2017/03/27 23:15:09]

It doesn't make a big difference, but this could also be written a little
differently:

    input_failure_types = {type(f) for f in failure_list}
    reftest_failure_types = {
        FailureReftestMismatch,
        FailureReftestMismatchDidNotOccur,
        FailureReftestNoImageGenerated,
        FailureReftestNoReferenceImageGenerated,
    }
    return bool(input_failure_types & reftest_failure_types)

WDYT?

[skobes, 2017/03/27 23:30:17]

Done.

 
 # FIXME: This is backwards.  Each TestFailure subclass should know what
 # test_expectation type it corresponds too.  Then this method just
 # collects them all from the failure list and returns the worst one.
 
 
 def determine_result_type(failure_list):
     """Takes a set of test_failures and returns which result type best fits
     the list of failures. "Best fits" means we use the worst type of failure.
 
@@ skipping 165 matching lines @@
 class FailureReftestMismatchDidNotOccur(TestFailure):
 
     def __init__(self, reference_filename=None):
         super(FailureReftestMismatchDidNotOccur, self).__init__()
         self.reference_filename = reference_filename
 
     def message(self):
         return "reference mismatch didn't happen"
 
 
-class FailureReftestNoImagesGenerated(TestFailure):
+class FailureReftestNoImageGenerated(TestFailure):
 
     def __init__(self, reference_filename=None):
-        super(FailureReftestNoImagesGenerated, self).__init__()
+        super(FailureReftestNoImageGenerated, self).__init__()
         self.reference_filename = reference_filename
 
     def message(self):
-        return "reference didn't generate pixel results."
+        return "reference test didn't generate pixel results"
+
+
+class FailureReftestNoReferenceImageGenerated(TestFailure):
+
+    def __init__(self, reference_filename=None):
+        super(FailureReftestNoReferenceImageGenerated, self).__init__()
+        self.reference_filename = reference_filename
+
+    def message(self):
+        return "-expected.html didn't generate pixel results"
 
 
 class FailureMissingAudio(TestFailure):
 
     def message(self):
         return 'expected audio result was missing'
 
 
 class FailureAudioMismatch(TestFailure):
 
     def message(self):
         return 'audio mismatch'
 
 
 class FailureEarlyExit(TestFailure):
 
     def message(self):
         return 'skipped due to early exit'
 
 
 # Convenient collection of all failure classes for anything that might
 # need to enumerate over them all.
 ALL_FAILURE_CLASSES = (FailureTimeout, FailureCrash, FailureMissingResult,
                        FailureTestHarnessAssertion,
                        FailureTextMismatch, FailureMissingImageHash,
                        FailureMissingImage, FailureImageHashMismatch,
                        FailureImageHashIncorrect, FailureReftestMismatch,
-                       FailureReftestMismatchDidNotOccur, FailureReftestNoImagesGenerated,
+                       FailureReftestMismatchDidNotOccur,
+                       FailureReftestNoImageGenerated,
+                       FailureReftestNoReferenceImageGenerated,
                        FailureMissingAudio, FailureAudioMismatch,
                        FailureEarlyExit)