| OLD | NEW |
| 1 # Copyright (C) 2010 Google Inc. All rights reserved. | 1 # Copyright (C) 2010 Google Inc. All rights reserved. |
| 2 # | 2 # |
| 3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
| 4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
| 5 # met: | 5 # met: |
| 6 # | 6 # |
| 7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
| 8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
| 9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
| 10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
| (...skipping 15 matching lines...) Expand all Loading... |
| 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 28 | 28 |
| 29 import cPickle | 29 import cPickle |
| 30 | 30 |
| 31 from webkitpy.layout_tests.models import test_expectations | 31 from webkitpy.layout_tests.models import test_expectations |
| 32 | 32 |
| 33 | 33 |
| 34 def is_reftest_failure(failure_list): | 34 def is_reftest_failure(failure_list): |
| 35 failure_types = [type(f) for f in failure_list] | 35 failure_types = [type(f) for f in failure_list] |
| 36 return set((FailureReftestMismatch, FailureReftestMismatchDidNotOccur, Failu
reReftestNoImagesGenerated)).intersection(failure_types) | 36 return set((FailureReftestMismatch, FailureReftestMismatchDidNotOccur, Failu
reReftestNoImagesGenerated)).intersection( |
| 37 failure_types) |
| 37 | 38 |
| 38 # FIXME: This is backwards. Each TestFailure subclass should know what | 39 # FIXME: This is backwards. Each TestFailure subclass should know what |
| 39 # test_expectation type it corresponds too. Then this method just | 40 # test_expectation type it corresponds too. Then this method just |
| 40 # collects them all from the failure list and returns the worst one. | 41 # collects them all from the failure list and returns the worst one. |
| 42 |
| 43 |
| 41 def determine_result_type(failure_list): | 44 def determine_result_type(failure_list): |
| 42 """Takes a set of test_failures and returns which result type best fits | 45 """Takes a set of test_failures and returns which result type best fits |
| 43 the list of failures. "Best fits" means we use the worst type of failure. | 46 the list of failures. "Best fits" means we use the worst type of failure. |
| 44 | 47 |
| 45 Returns: | 48 Returns: |
| 46 one of the test_expectations result types - PASS, FAIL, CRASH, etc.""" | 49 one of the test_expectations result types - PASS, FAIL, CRASH, etc.""" |
| 47 | 50 |
| 48 if not failure_list or len(failure_list) == 0: | 51 if not failure_list or len(failure_list) == 0: |
| 49 return test_expectations.PASS | 52 return test_expectations.PASS |
| 50 | 53 |
| (...skipping 19 matching lines...) Expand all Loading... |
| 70 is_audio_failure = (FailureAudioMismatch in failure_types) | 73 is_audio_failure = (FailureAudioMismatch in failure_types) |
| 71 if is_text_failure and is_image_failure: | 74 if is_text_failure and is_image_failure: |
| 72 return test_expectations.IMAGE_PLUS_TEXT | 75 return test_expectations.IMAGE_PLUS_TEXT |
| 73 elif is_text_failure: | 76 elif is_text_failure: |
| 74 return test_expectations.TEXT | 77 return test_expectations.TEXT |
| 75 elif is_image_failure or is_reftest_failure(failure_list): | 78 elif is_image_failure or is_reftest_failure(failure_list): |
| 76 return test_expectations.IMAGE | 79 return test_expectations.IMAGE |
| 77 elif is_audio_failure: | 80 elif is_audio_failure: |
| 78 return test_expectations.AUDIO | 81 return test_expectations.AUDIO |
| 79 else: | 82 else: |
| 80 raise ValueError("unclassifiable set of failures: " | 83 raise ValueError('unclassifiable set of failures: ' |
| 81 + str(failure_types)) | 84 + str(failure_types)) |
| 82 | 85 |
| 83 | 86 |
| 84 class TestFailure(object): | 87 class TestFailure(object): |
| 88 |
| 85 """Abstract base class that defines the failure interface.""" | 89 """Abstract base class that defines the failure interface.""" |
| 86 | 90 |
| 87 @staticmethod | 91 @staticmethod |
| 88 def loads(s): | 92 def loads(s): |
| 89 """Creates a TestFailure object from the specified string.""" | 93 """Creates a TestFailure object from the specified string.""" |
| 90 return cPickle.loads(s) | 94 return cPickle.loads(s) |
| 91 | 95 |
| 92 def message(self): | 96 def message(self): |
| 93 """Returns a string describing the failure in more detail.""" | 97 """Returns a string describing the failure in more detail.""" |
| 94 raise NotImplementedError | 98 raise NotImplementedError |
| (...skipping 10 matching lines...) Expand all Loading... |
| 105 def dumps(self): | 109 def dumps(self): |
| 106 """Returns the string/JSON representation of a TestFailure.""" | 110 """Returns the string/JSON representation of a TestFailure.""" |
| 107 return cPickle.dumps(self) | 111 return cPickle.dumps(self) |
| 108 | 112 |
| 109 def driver_needs_restart(self): | 113 def driver_needs_restart(self): |
| 110 """Returns True if we should kill the driver before the next test.""" | 114 """Returns True if we should kill the driver before the next test.""" |
| 111 return False | 115 return False |
| 112 | 116 |
| 113 | 117 |
| 114 class FailureTimeout(TestFailure): | 118 class FailureTimeout(TestFailure): |
| 119 |
| 115 def __init__(self, is_reftest=False): | 120 def __init__(self, is_reftest=False): |
| 116 super(FailureTimeout, self).__init__() | 121 super(FailureTimeout, self).__init__() |
| 117 self.is_reftest = is_reftest | 122 self.is_reftest = is_reftest |
| 118 | 123 |
| 119 def message(self): | 124 def message(self): |
| 120 return "test timed out" | 125 return 'test timed out' |
| 121 | 126 |
| 122 def driver_needs_restart(self): | 127 def driver_needs_restart(self): |
| 123 return True | 128 return True |
| 124 | 129 |
| 125 | 130 |
| 126 class FailureCrash(TestFailure): | 131 class FailureCrash(TestFailure): |
| 132 |
| 127 def __init__(self, is_reftest=False, process_name='content_shell', pid=None)
: | 133 def __init__(self, is_reftest=False, process_name='content_shell', pid=None)
: |
| 128 super(FailureCrash, self).__init__() | 134 super(FailureCrash, self).__init__() |
| 129 self.process_name = process_name | 135 self.process_name = process_name |
| 130 self.pid = pid | 136 self.pid = pid |
| 131 self.is_reftest = is_reftest | 137 self.is_reftest = is_reftest |
| 132 | 138 |
| 133 def message(self): | 139 def message(self): |
| 134 if self.pid: | 140 if self.pid: |
| 135 return "%s crashed [pid=%d]" % (self.process_name, self.pid) | 141 return '%s crashed [pid=%d]' % (self.process_name, self.pid) |
| 136 return self.process_name + " crashed" | 142 return self.process_name + ' crashed' |
| 137 | 143 |
| 138 def driver_needs_restart(self): | 144 def driver_needs_restart(self): |
| 139 return True | 145 return True |
| 140 | 146 |
| 141 | 147 |
| 142 class FailureLeak(TestFailure): | 148 class FailureLeak(TestFailure): |
| 149 |
| 143 def __init__(self, is_reftest=False, log=''): | 150 def __init__(self, is_reftest=False, log=''): |
| 144 super(FailureLeak, self).__init__() | 151 super(FailureLeak, self).__init__() |
| 145 self.is_reftest = is_reftest | 152 self.is_reftest = is_reftest |
| 146 self.log = log | 153 self.log = log |
| 147 | 154 |
| 148 def message(self): | 155 def message(self): |
| 149 return "leak detected: %s" % (self.log) | 156 return 'leak detected: %s' % (self.log) |
| 150 | 157 |
| 151 | 158 |
| 152 class FailureMissingResult(TestFailure): | 159 class FailureMissingResult(TestFailure): |
| 160 |
| 153 def message(self): | 161 def message(self): |
| 154 return "-expected.txt was missing" | 162 return '-expected.txt was missing' |
| 155 | 163 |
| 156 | 164 |
| 157 class FailureTestHarnessAssertion(TestFailure): | 165 class FailureTestHarnessAssertion(TestFailure): |
| 166 |
| 158 def message(self): | 167 def message(self): |
| 159 return "asserts failed" | 168 return 'asserts failed' |
| 160 | 169 |
| 161 | 170 |
| 162 class FailureTextMismatch(TestFailure): | 171 class FailureTextMismatch(TestFailure): |
| 172 |
| 163 def message(self): | 173 def message(self): |
| 164 return "text diff" | 174 return 'text diff' |
| 165 | 175 |
| 166 | 176 |
| 167 class FailureMissingImageHash(TestFailure): | 177 class FailureMissingImageHash(TestFailure): |
| 178 |
| 168 def message(self): | 179 def message(self): |
| 169 return "-expected.png was missing an embedded checksum" | 180 return '-expected.png was missing an embedded checksum' |
| 170 | 181 |
| 171 | 182 |
| 172 class FailureMissingImage(TestFailure): | 183 class FailureMissingImage(TestFailure): |
| 184 |
| 173 def message(self): | 185 def message(self): |
| 174 return "-expected.png was missing" | 186 return '-expected.png was missing' |
| 175 | 187 |
| 176 | 188 |
| 177 class FailureImageHashMismatch(TestFailure): | 189 class FailureImageHashMismatch(TestFailure): |
| 190 |
| 178 def message(self): | 191 def message(self): |
| 179 return "image diff" | 192 return 'image diff' |
| 180 | 193 |
| 181 | 194 |
| 182 class FailureImageHashIncorrect(TestFailure): | 195 class FailureImageHashIncorrect(TestFailure): |
| 196 |
| 183 def message(self): | 197 def message(self): |
| 184 return "-expected.png embedded checksum is incorrect" | 198 return '-expected.png embedded checksum is incorrect' |
| 185 | 199 |
| 186 | 200 |
| 187 class FailureReftestMismatch(TestFailure): | 201 class FailureReftestMismatch(TestFailure): |
| 202 |
| 188 def __init__(self, reference_filename=None): | 203 def __init__(self, reference_filename=None): |
| 189 super(FailureReftestMismatch, self).__init__() | 204 super(FailureReftestMismatch, self).__init__() |
| 190 self.reference_filename = reference_filename | 205 self.reference_filename = reference_filename |
| 191 | 206 |
| 192 def message(self): | 207 def message(self): |
| 193 return "reference mismatch" | 208 return 'reference mismatch' |
| 194 | 209 |
| 195 | 210 |
| 196 class FailureReftestMismatchDidNotOccur(TestFailure): | 211 class FailureReftestMismatchDidNotOccur(TestFailure): |
| 212 |
| 197 def __init__(self, reference_filename=None): | 213 def __init__(self, reference_filename=None): |
| 198 super(FailureReftestMismatchDidNotOccur, self).__init__() | 214 super(FailureReftestMismatchDidNotOccur, self).__init__() |
| 199 self.reference_filename = reference_filename | 215 self.reference_filename = reference_filename |
| 200 | 216 |
| 201 def message(self): | 217 def message(self): |
| 202 return "reference mismatch didn't happen" | 218 return "reference mismatch didn't happen" |
| 203 | 219 |
| 204 | 220 |
| 205 class FailureReftestNoImagesGenerated(TestFailure): | 221 class FailureReftestNoImagesGenerated(TestFailure): |
| 222 |
| 206 def __init__(self, reference_filename=None): | 223 def __init__(self, reference_filename=None): |
| 207 super(FailureReftestNoImagesGenerated, self).__init__() | 224 super(FailureReftestNoImagesGenerated, self).__init__() |
| 208 self.reference_filename = reference_filename | 225 self.reference_filename = reference_filename |
| 209 | 226 |
| 210 def message(self): | 227 def message(self): |
| 211 return "reference didn't generate pixel results." | 228 return "reference didn't generate pixel results." |
| 212 | 229 |
| 213 | 230 |
| 214 class FailureMissingAudio(TestFailure): | 231 class FailureMissingAudio(TestFailure): |
| 232 |
| 215 def message(self): | 233 def message(self): |
| 216 return "expected audio result was missing" | 234 return 'expected audio result was missing' |
| 217 | 235 |
| 218 | 236 |
| 219 class FailureAudioMismatch(TestFailure): | 237 class FailureAudioMismatch(TestFailure): |
| 238 |
| 220 def message(self): | 239 def message(self): |
| 221 return "audio mismatch" | 240 return 'audio mismatch' |
| 222 | 241 |
| 223 | 242 |
| 224 class FailureEarlyExit(TestFailure): | 243 class FailureEarlyExit(TestFailure): |
| 244 |
| 225 def message(self): | 245 def message(self): |
| 226 return "skipped due to early exit" | 246 return 'skipped due to early exit' |
| 227 | 247 |
| 228 | 248 |
| 229 # Convenient collection of all failure classes for anything that might | 249 # Convenient collection of all failure classes for anything that might |
| 230 # need to enumerate over them all. | 250 # need to enumerate over them all. |
| 231 ALL_FAILURE_CLASSES = (FailureTimeout, FailureCrash, FailureMissingResult, | 251 ALL_FAILURE_CLASSES = (FailureTimeout, FailureCrash, FailureMissingResult, |
| 232 FailureTestHarnessAssertion, | 252 FailureTestHarnessAssertion, |
| 233 FailureTextMismatch, FailureMissingImageHash, | 253 FailureTextMismatch, FailureMissingImageHash, |
| 234 FailureMissingImage, FailureImageHashMismatch, | 254 FailureMissingImage, FailureImageHashMismatch, |
| 235 FailureImageHashIncorrect, FailureReftestMismatch, | 255 FailureImageHashIncorrect, FailureReftestMismatch, |
| 236 FailureReftestMismatchDidNotOccur, FailureReftestNoImages
Generated, | 256 FailureReftestMismatchDidNotOccur, FailureReftestNoImages
Generated, |
| 237 FailureMissingAudio, FailureAudioMismatch, | 257 FailureMissingAudio, FailureAudioMismatch, |
| 238 FailureEarlyExit) | 258 FailureEarlyExit) |
| OLD | NEW |