OLD | NEW |
1 # Copyright (C) 2010 Google Inc. All rights reserved. | 1 # Copyright (C) 2010 Google Inc. All rights reserved. |
2 # | 2 # |
3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
5 # met: | 5 # met: |
6 # | 6 # |
7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
(...skipping 85 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
96 | 96 |
97 def __contains__(self, item): | 97 def __contains__(self, item): |
98 return item in self.tests | 98 return item in self.tests |
99 | 99 |
100 def __getitem__(self, item): | 100 def __getitem__(self, item): |
101 return self.tests[item] | 101 return self.tests[item] |
102 | 102 |
103 # | 103 # |
104 # These numbers may need to be updated whenever we add or delete tests. This inc
ludes virtual tests. | 104 # These numbers may need to be updated whenever we add or delete tests. This inc
ludes virtual tests. |
105 # | 105 # |
106 TOTAL_TESTS = 117 | 106 TOTAL_TESTS = 113 |
107 TOTAL_SKIPS = 30 | 107 TOTAL_SKIPS = 29 |
108 | 108 |
109 UNEXPECTED_PASSES = 1 | 109 UNEXPECTED_PASSES = 1 |
110 UNEXPECTED_FAILURES = 26 | 110 UNEXPECTED_FAILURES = 26 |
111 | 111 |
112 def unit_test_list(): | 112 def unit_test_list(): |
113 tests = TestList() | 113 tests = TestList() |
114 tests.add('failures/expected/crash.html', crash=True) | 114 tests.add('failures/expected/crash.html', crash=True) |
115 tests.add('failures/expected/exception.html', exception=True) | 115 tests.add('failures/expected/exception.html', exception=True) |
116 tests.add('failures/expected/device_failure.html', device_failure=True) | 116 tests.add('failures/expected/device_failure.html', device_failure=True) |
117 tests.add('failures/expected/timeout.html', timeout=True) | 117 tests.add('failures/expected/timeout.html', timeout=True) |
(...skipping 20 matching lines...) Expand all Loading... |
138 actual_text=None, expected_text=None, | 138 actual_text=None, expected_text=None, |
139 actual_image=None, expected_image=None, | 139 actual_image=None, expected_image=None, |
140 actual_checksum=None) | 140 actual_checksum=None) |
141 tests.add('failures/expected/missing_text.html', expected_text=None) | 141 tests.add('failures/expected/missing_text.html', expected_text=None) |
142 tests.add('failures/expected/newlines_leading.html', | 142 tests.add('failures/expected/newlines_leading.html', |
143 expected_text="\nfoo\n", actual_text="foo\n") | 143 expected_text="\nfoo\n", actual_text="foo\n") |
144 tests.add('failures/expected/newlines_trailing.html', | 144 tests.add('failures/expected/newlines_trailing.html', |
145 expected_text="foo\n\n", actual_text="foo\n") | 145 expected_text="foo\n\n", actual_text="foo\n") |
146 tests.add('failures/expected/newlines_with_excess_CR.html', | 146 tests.add('failures/expected/newlines_with_excess_CR.html', |
147 expected_text="foo\r\r\r\n", actual_text="foo\n") | 147 expected_text="foo\r\r\r\n", actual_text="foo\n") |
148 tests.add('failures/expected/testharness.html', | |
149 actual_text='This is a testharness.js-based test.\nFAIL: assert fire
d\n.Harness: the test ran to completion.\n\n', expected_text=None, | |
150 actual_image=None, expected_image=None, | |
151 actual_checksum=None) | |
152 tests.add('failures/expected/testharness.html', | |
153 actual_text='RANDOM TEXT.\nThis is a testharness.js-based test.\nPAS
S: things are fine.\n.Harness: the test ran to completion.\n\n', expected_text=N
one, | |
154 actual_image=None, expected_image=None, | |
155 actual_checksum=None) | |
156 tests.add('failures/expected/text.html', actual_text='text_fail-png') | 148 tests.add('failures/expected/text.html', actual_text='text_fail-png') |
157 tests.add('failures/expected/crash_then_text.html') | 149 tests.add('failures/expected/crash_then_text.html') |
158 tests.add('failures/expected/skip_text.html', actual_text='text diff') | 150 tests.add('failures/expected/skip_text.html', actual_text='text diff') |
159 tests.add('failures/flaky/text.html') | 151 tests.add('failures/flaky/text.html') |
160 tests.add('failures/unexpected/missing_text.html', expected_text=None) | 152 tests.add('failures/unexpected/missing_text.html', expected_text=None) |
161 tests.add('failures/unexpected/missing_check.html', expected_image='missing-
check-png') | 153 tests.add('failures/unexpected/missing_check.html', expected_image='missing-
check-png') |
162 tests.add('failures/unexpected/missing_image.html', expected_image=None) | 154 tests.add('failures/unexpected/missing_image.html', expected_image=None) |
163 tests.add('failures/unexpected/missing_render_tree_dump.html', actual_text="
""layer at (0,0) size 800x600 | 155 tests.add('failures/unexpected/missing_render_tree_dump.html', actual_text="
""layer at (0,0) size 800x600 |
164 RenderView at (0,0) size 800x600 | 156 RenderView at (0,0) size 800x600 |
165 layer at (0,0) size 800x34 | 157 layer at (0,0) size 800x34 |
(...skipping 30 matching lines...) Expand all Loading... |
196 tests.add('passes/image.html') | 188 tests.add('passes/image.html') |
197 tests.add('passes/audio.html', | 189 tests.add('passes/audio.html', |
198 actual_audio=base64.b64encode('audio-wav'), expected_audio='audio-
wav', | 190 actual_audio=base64.b64encode('audio-wav'), expected_audio='audio-
wav', |
199 actual_text=None, expected_text=None, | 191 actual_text=None, expected_text=None, |
200 actual_image=None, expected_image=None, | 192 actual_image=None, expected_image=None, |
201 actual_checksum=None) | 193 actual_checksum=None) |
202 tests.add('passes/platform_image.html') | 194 tests.add('passes/platform_image.html') |
203 tests.add('passes/checksum_in_image.html', | 195 tests.add('passes/checksum_in_image.html', |
204 expected_image='tEXtchecksum\x00checksum_in_image-checksum') | 196 expected_image='tEXtchecksum\x00checksum_in_image-checksum') |
205 tests.add('passes/skipped/skip.html') | 197 tests.add('passes/skipped/skip.html') |
206 tests.add('passes/testharness.html', | |
207 actual_text='CONSOLE LOG: error.\nThis is a testharness.js-based tes
t.\nPASS: things are fine.\n.Harness: the test ran to completion.\n\n', expected
_text=None, | |
208 actual_image=None, expected_image=None, | |
209 actual_checksum=None) | |
210 tests.add('passes/testharness.html', | |
211 actual_text='CONSOLE ERROR: error.\nThis is a testharness.js-based t
est.\nPASS: things are fine.\n.Harness: the test ran to completion.\n\n', expect
ed_text=None, | |
212 actual_image=None, expected_image=None, | |
213 actual_checksum=None) | |
214 tests.add('passes/testharness.html', | |
215 actual_text=' This is a testharness.js-based test.\nPASS: assert is
fine\nHarness: the test ran to completion.\n\n', expected_text=None, | |
216 actual_image=None, expected_image=None, | |
217 actual_checksum=None) | |
218 tests.add('passes/testharness.html', | |
219 actual_text='This is a testharness.js-based test.\nPASS: assert is f
ine\nHarness: the test ran to completion.\n\n', expected_text=None, | |
220 actual_image=None, expected_image=None, | |
221 actual_checksum=None) | |
222 | 198 |
223 # Note that here the checksums don't match but the images do, so this test p
asses "unexpectedly". | 199 # Note that here the checksums don't match but the images do, so this test p
asses "unexpectedly". |
224 # See https://bugs.webkit.org/show_bug.cgi?id=69444 . | 200 # See https://bugs.webkit.org/show_bug.cgi?id=69444 . |
225 tests.add('failures/unexpected/checksum.html', actual_checksum='checksum_fai
l-checksum') | 201 tests.add('failures/unexpected/checksum.html', actual_checksum='checksum_fai
l-checksum') |
226 | 202 |
227 # Text output files contain "\r\n" on Windows. This may be | 203 # Text output files contain "\r\n" on Windows. This may be |
228 # helpfully filtered to "\r\r\n" by our Python/Cygwin tooling. | 204 # helpfully filtered to "\r\r\n" by our Python/Cygwin tooling. |
229 tests.add('passes/text.html', | 205 tests.add('passes/text.html', |
230 expected_text='\nfoo\n\n', actual_text='\nfoo\r\n\r\r\n') | 206 expected_text='\nfoo\n\n', actual_text='\nfoo\r\n\r\r\n') |
231 | 207 |
(...skipping 85 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
317 Bug(test) failures/expected/mismatch.html [ ImageOnlyFailure ] | 293 Bug(test) failures/expected/mismatch.html [ ImageOnlyFailure ] |
318 Bug(test) failures/expected/missing_check.html [ Missing Pass ] | 294 Bug(test) failures/expected/missing_check.html [ Missing Pass ] |
319 Bug(test) failures/expected/missing_image.html [ Missing Pass ] | 295 Bug(test) failures/expected/missing_image.html [ Missing Pass ] |
320 Bug(test) failures/expected/missing_audio.html [ Missing Pass ] | 296 Bug(test) failures/expected/missing_audio.html [ Missing Pass ] |
321 Bug(test) failures/expected/missing_text.html [ Missing Pass ] | 297 Bug(test) failures/expected/missing_text.html [ Missing Pass ] |
322 Bug(test) failures/expected/newlines_leading.html [ Failure ] | 298 Bug(test) failures/expected/newlines_leading.html [ Failure ] |
323 Bug(test) failures/expected/newlines_trailing.html [ Failure ] | 299 Bug(test) failures/expected/newlines_trailing.html [ Failure ] |
324 Bug(test) failures/expected/newlines_with_excess_CR.html [ Failure ] | 300 Bug(test) failures/expected/newlines_with_excess_CR.html [ Failure ] |
325 Bug(test) failures/expected/reftest.html [ ImageOnlyFailure ] | 301 Bug(test) failures/expected/reftest.html [ ImageOnlyFailure ] |
326 Bug(test) failures/expected/text.html [ Failure ] | 302 Bug(test) failures/expected/text.html [ Failure ] |
327 Bug(test) failures/expected/testharness.html [ Failure ] | |
328 Bug(test) failures/expected/timeout.html [ Timeout ] | 303 Bug(test) failures/expected/timeout.html [ Timeout ] |
329 Bug(test) failures/expected/keyboard.html [ WontFix ] | 304 Bug(test) failures/expected/keyboard.html [ WontFix ] |
330 Bug(test) failures/expected/exception.html [ WontFix ] | 305 Bug(test) failures/expected/exception.html [ WontFix ] |
331 Bug(test) failures/expected/device_failure.html [ WontFix ] | 306 Bug(test) failures/expected/device_failure.html [ WontFix ] |
332 Bug(test) failures/expected/leak.html [ Leak ] | 307 Bug(test) failures/expected/leak.html [ Leak ] |
333 Bug(test) failures/unexpected/pass.html [ Failure ] | 308 Bug(test) failures/unexpected/pass.html [ Failure ] |
334 Bug(test) passes/skipped/skip.html [ Skip ] | 309 Bug(test) passes/skipped/skip.html [ Skip ] |
335 Bug(test) passes/text.html [ Pass ] | 310 Bug(test) passes/text.html [ Pass ] |
336 """) | 311 """) |
337 | 312 |
(...skipping 324 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
662 else: | 637 else: |
663 image = test.actual_image | 638 image = test.actual_image |
664 return DriverOutput(actual_text, image, test.actual_checksum, audio, | 639 return DriverOutput(actual_text, image, test.actual_checksum, audio, |
665 crash=(crash or web_process_crash), crashed_process_name=crashed_pro
cess_name, | 640 crash=(crash or web_process_crash), crashed_process_name=crashed_pro
cess_name, |
666 crashed_pid=crashed_pid, crash_log=crash_log, | 641 crashed_pid=crashed_pid, crash_log=crash_log, |
667 test_time=time.time() - start_time, timeout=test.timeout, error=test
.error, pid=self.pid, | 642 test_time=time.time() - start_time, timeout=test.timeout, error=test
.error, pid=self.pid, |
668 leak=test.leak) | 643 leak=test.leak) |
669 | 644 |
670 def stop(self): | 645 def stop(self): |
671 self.started = False | 646 self.started = False |
OLD | NEW |