OLD | NEW |
1 # Copyright (C) 2010 Google Inc. All rights reserved. | 1 # Copyright (C) 2010 Google Inc. All rights reserved. |
2 # | 2 # |
3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
5 # met: | 5 # met: |
6 # | 6 # |
7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
(...skipping 15 matching lines...) Expand all Loading... |
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
28 | 28 |
29 import cPickle | 29 import cPickle |
30 | 30 |
31 from webkitpy.layout_tests.models import test_expectations | 31 from webkitpy.layout_tests.models import test_expectations |
32 | 32 |
33 | 33 |
34 def is_reftest_failure(failure_list): | 34 def is_reftest_failure(failure_list): |
35 failure_types = [type(f) for f in failure_list] | 35 failure_types = [type(f) for f in failure_list] |
36 return set((FailureReftestMismatch, FailureReftestMismatchDidNotOccur, Failu
reReftestNoImagesGenerated)).intersection(failure_types) | 36 return set((FailureReftestMismatch, FailureReftestMismatchDidNotOccur, Failu
reReftestNoImagesGenerated)).intersection( |
| 37 failure_types) |
37 | 38 |
38 # FIXME: This is backwards. Each TestFailure subclass should know what | 39 # FIXME: This is backwards. Each TestFailure subclass should know what |
39 # test_expectation type it corresponds too. Then this method just | 40 # test_expectation type it corresponds too. Then this method just |
40 # collects them all from the failure list and returns the worst one. | 41 # collects them all from the failure list and returns the worst one. |
| 42 |
| 43 |
41 def determine_result_type(failure_list): | 44 def determine_result_type(failure_list): |
42 """Takes a set of test_failures and returns which result type best fits | 45 """Takes a set of test_failures and returns which result type best fits |
43 the list of failures. "Best fits" means we use the worst type of failure. | 46 the list of failures. "Best fits" means we use the worst type of failure. |
44 | 47 |
45 Returns: | 48 Returns: |
46 one of the test_expectations result types - PASS, FAIL, CRASH, etc.""" | 49 one of the test_expectations result types - PASS, FAIL, CRASH, etc.""" |
47 | 50 |
48 if not failure_list or len(failure_list) == 0: | 51 if not failure_list or len(failure_list) == 0: |
49 return test_expectations.PASS | 52 return test_expectations.PASS |
50 | 53 |
(...skipping 19 matching lines...) Expand all Loading... |
70 is_audio_failure = (FailureAudioMismatch in failure_types) | 73 is_audio_failure = (FailureAudioMismatch in failure_types) |
71 if is_text_failure and is_image_failure: | 74 if is_text_failure and is_image_failure: |
72 return test_expectations.IMAGE_PLUS_TEXT | 75 return test_expectations.IMAGE_PLUS_TEXT |
73 elif is_text_failure: | 76 elif is_text_failure: |
74 return test_expectations.TEXT | 77 return test_expectations.TEXT |
75 elif is_image_failure or is_reftest_failure(failure_list): | 78 elif is_image_failure or is_reftest_failure(failure_list): |
76 return test_expectations.IMAGE | 79 return test_expectations.IMAGE |
77 elif is_audio_failure: | 80 elif is_audio_failure: |
78 return test_expectations.AUDIO | 81 return test_expectations.AUDIO |
79 else: | 82 else: |
80 raise ValueError("unclassifiable set of failures: " | 83 raise ValueError('unclassifiable set of failures: ' |
81 + str(failure_types)) | 84 + str(failure_types)) |
82 | 85 |
83 | 86 |
84 class TestFailure(object): | 87 class TestFailure(object): |
| 88 |
85 """Abstract base class that defines the failure interface.""" | 89 """Abstract base class that defines the failure interface.""" |
86 | 90 |
87 @staticmethod | 91 @staticmethod |
88 def loads(s): | 92 def loads(s): |
89 """Creates a TestFailure object from the specified string.""" | 93 """Creates a TestFailure object from the specified string.""" |
90 return cPickle.loads(s) | 94 return cPickle.loads(s) |
91 | 95 |
92 def message(self): | 96 def message(self): |
93 """Returns a string describing the failure in more detail.""" | 97 """Returns a string describing the failure in more detail.""" |
94 raise NotImplementedError | 98 raise NotImplementedError |
(...skipping 10 matching lines...) Expand all Loading... |
105 def dumps(self): | 109 def dumps(self): |
106 """Returns the string/JSON representation of a TestFailure.""" | 110 """Returns the string/JSON representation of a TestFailure.""" |
107 return cPickle.dumps(self) | 111 return cPickle.dumps(self) |
108 | 112 |
109 def driver_needs_restart(self): | 113 def driver_needs_restart(self): |
110 """Returns True if we should kill the driver before the next test.""" | 114 """Returns True if we should kill the driver before the next test.""" |
111 return False | 115 return False |
112 | 116 |
113 | 117 |
114 class FailureTimeout(TestFailure): | 118 class FailureTimeout(TestFailure): |
| 119 |
115 def __init__(self, is_reftest=False): | 120 def __init__(self, is_reftest=False): |
116 super(FailureTimeout, self).__init__() | 121 super(FailureTimeout, self).__init__() |
117 self.is_reftest = is_reftest | 122 self.is_reftest = is_reftest |
118 | 123 |
119 def message(self): | 124 def message(self): |
120 return "test timed out" | 125 return 'test timed out' |
121 | 126 |
122 def driver_needs_restart(self): | 127 def driver_needs_restart(self): |
123 return True | 128 return True |
124 | 129 |
125 | 130 |
126 class FailureCrash(TestFailure): | 131 class FailureCrash(TestFailure): |
| 132 |
127 def __init__(self, is_reftest=False, process_name='content_shell', pid=None)
: | 133 def __init__(self, is_reftest=False, process_name='content_shell', pid=None)
: |
128 super(FailureCrash, self).__init__() | 134 super(FailureCrash, self).__init__() |
129 self.process_name = process_name | 135 self.process_name = process_name |
130 self.pid = pid | 136 self.pid = pid |
131 self.is_reftest = is_reftest | 137 self.is_reftest = is_reftest |
132 | 138 |
133 def message(self): | 139 def message(self): |
134 if self.pid: | 140 if self.pid: |
135 return "%s crashed [pid=%d]" % (self.process_name, self.pid) | 141 return '%s crashed [pid=%d]' % (self.process_name, self.pid) |
136 return self.process_name + " crashed" | 142 return self.process_name + ' crashed' |
137 | 143 |
138 def driver_needs_restart(self): | 144 def driver_needs_restart(self): |
139 return True | 145 return True |
140 | 146 |
141 | 147 |
142 class FailureLeak(TestFailure): | 148 class FailureLeak(TestFailure): |
| 149 |
143 def __init__(self, is_reftest=False, log=''): | 150 def __init__(self, is_reftest=False, log=''): |
144 super(FailureLeak, self).__init__() | 151 super(FailureLeak, self).__init__() |
145 self.is_reftest = is_reftest | 152 self.is_reftest = is_reftest |
146 self.log = log | 153 self.log = log |
147 | 154 |
148 def message(self): | 155 def message(self): |
149 return "leak detected: %s" % (self.log) | 156 return 'leak detected: %s' % (self.log) |
150 | 157 |
151 | 158 |
152 class FailureMissingResult(TestFailure): | 159 class FailureMissingResult(TestFailure): |
| 160 |
153 def message(self): | 161 def message(self): |
154 return "-expected.txt was missing" | 162 return '-expected.txt was missing' |
155 | 163 |
156 | 164 |
157 class FailureTestHarnessAssertion(TestFailure): | 165 class FailureTestHarnessAssertion(TestFailure): |
| 166 |
158 def message(self): | 167 def message(self): |
159 return "asserts failed" | 168 return 'asserts failed' |
160 | 169 |
161 | 170 |
162 class FailureTextMismatch(TestFailure): | 171 class FailureTextMismatch(TestFailure): |
| 172 |
163 def message(self): | 173 def message(self): |
164 return "text diff" | 174 return 'text diff' |
165 | 175 |
166 | 176 |
167 class FailureMissingImageHash(TestFailure): | 177 class FailureMissingImageHash(TestFailure): |
| 178 |
168 def message(self): | 179 def message(self): |
169 return "-expected.png was missing an embedded checksum" | 180 return '-expected.png was missing an embedded checksum' |
170 | 181 |
171 | 182 |
172 class FailureMissingImage(TestFailure): | 183 class FailureMissingImage(TestFailure): |
| 184 |
173 def message(self): | 185 def message(self): |
174 return "-expected.png was missing" | 186 return '-expected.png was missing' |
175 | 187 |
176 | 188 |
177 class FailureImageHashMismatch(TestFailure): | 189 class FailureImageHashMismatch(TestFailure): |
| 190 |
178 def message(self): | 191 def message(self): |
179 return "image diff" | 192 return 'image diff' |
180 | 193 |
181 | 194 |
182 class FailureImageHashIncorrect(TestFailure): | 195 class FailureImageHashIncorrect(TestFailure): |
| 196 |
183 def message(self): | 197 def message(self): |
184 return "-expected.png embedded checksum is incorrect" | 198 return '-expected.png embedded checksum is incorrect' |
185 | 199 |
186 | 200 |
187 class FailureReftestMismatch(TestFailure): | 201 class FailureReftestMismatch(TestFailure): |
| 202 |
188 def __init__(self, reference_filename=None): | 203 def __init__(self, reference_filename=None): |
189 super(FailureReftestMismatch, self).__init__() | 204 super(FailureReftestMismatch, self).__init__() |
190 self.reference_filename = reference_filename | 205 self.reference_filename = reference_filename |
191 | 206 |
192 def message(self): | 207 def message(self): |
193 return "reference mismatch" | 208 return 'reference mismatch' |
194 | 209 |
195 | 210 |
196 class FailureReftestMismatchDidNotOccur(TestFailure): | 211 class FailureReftestMismatchDidNotOccur(TestFailure): |
| 212 |
197 def __init__(self, reference_filename=None): | 213 def __init__(self, reference_filename=None): |
198 super(FailureReftestMismatchDidNotOccur, self).__init__() | 214 super(FailureReftestMismatchDidNotOccur, self).__init__() |
199 self.reference_filename = reference_filename | 215 self.reference_filename = reference_filename |
200 | 216 |
201 def message(self): | 217 def message(self): |
202 return "reference mismatch didn't happen" | 218 return "reference mismatch didn't happen" |
203 | 219 |
204 | 220 |
205 class FailureReftestNoImagesGenerated(TestFailure): | 221 class FailureReftestNoImagesGenerated(TestFailure): |
| 222 |
206 def __init__(self, reference_filename=None): | 223 def __init__(self, reference_filename=None): |
207 super(FailureReftestNoImagesGenerated, self).__init__() | 224 super(FailureReftestNoImagesGenerated, self).__init__() |
208 self.reference_filename = reference_filename | 225 self.reference_filename = reference_filename |
209 | 226 |
210 def message(self): | 227 def message(self): |
211 return "reference didn't generate pixel results." | 228 return "reference didn't generate pixel results." |
212 | 229 |
213 | 230 |
214 class FailureMissingAudio(TestFailure): | 231 class FailureMissingAudio(TestFailure): |
| 232 |
215 def message(self): | 233 def message(self): |
216 return "expected audio result was missing" | 234 return 'expected audio result was missing' |
217 | 235 |
218 | 236 |
219 class FailureAudioMismatch(TestFailure): | 237 class FailureAudioMismatch(TestFailure): |
| 238 |
220 def message(self): | 239 def message(self): |
221 return "audio mismatch" | 240 return 'audio mismatch' |
222 | 241 |
223 | 242 |
224 class FailureEarlyExit(TestFailure): | 243 class FailureEarlyExit(TestFailure): |
| 244 |
225 def message(self): | 245 def message(self): |
226 return "skipped due to early exit" | 246 return 'skipped due to early exit' |
227 | 247 |
228 | 248 |
229 # Convenient collection of all failure classes for anything that might | 249 # Convenient collection of all failure classes for anything that might |
230 # need to enumerate over them all. | 250 # need to enumerate over them all. |
231 ALL_FAILURE_CLASSES = (FailureTimeout, FailureCrash, FailureMissingResult, | 251 ALL_FAILURE_CLASSES = (FailureTimeout, FailureCrash, FailureMissingResult, |
232 FailureTestHarnessAssertion, | 252 FailureTestHarnessAssertion, |
233 FailureTextMismatch, FailureMissingImageHash, | 253 FailureTextMismatch, FailureMissingImageHash, |
234 FailureMissingImage, FailureImageHashMismatch, | 254 FailureMissingImage, FailureImageHashMismatch, |
235 FailureImageHashIncorrect, FailureReftestMismatch, | 255 FailureImageHashIncorrect, FailureReftestMismatch, |
236 FailureReftestMismatchDidNotOccur, FailureReftestNoImages
Generated, | 256 FailureReftestMismatchDidNotOccur, FailureReftestNoImages
Generated, |
237 FailureMissingAudio, FailureAudioMismatch, | 257 FailureMissingAudio, FailureAudioMismatch, |
238 FailureEarlyExit) | 258 FailureEarlyExit) |
OLD | NEW |