OLD | NEW |
---|---|
1 # Copyright (C) 2010 Google Inc. All rights reserved. | 1 # Copyright (C) 2010 Google Inc. All rights reserved. |
2 # | 2 # |
3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
5 # met: | 5 # met: |
6 # | 6 # |
7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
(...skipping 91 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
102 | 102 |
103 # | 103 # |
104 # These numbers may need to be updated whenever we add or delete tests. This inc ludes virtual tests. | 104 # These numbers may need to be updated whenever we add or delete tests. This inc ludes virtual tests. |
105 # | 105 # |
106 TOTAL_TESTS = 114 | 106 TOTAL_TESTS = 114 |
107 TOTAL_SKIPS = 26 | 107 TOTAL_SKIPS = 26 |
108 TOTAL_CRASHES = 80 | 108 TOTAL_CRASHES = 80 |
109 UNEXPECTED_PASSES = 1 | 109 UNEXPECTED_PASSES = 1 |
110 UNEXPECTED_FAILURES = 26 | 110 UNEXPECTED_FAILURES = 26 |
111 | 111 |
112 | |
112 def unit_test_list(): | 113 def unit_test_list(): |
113 tests = TestList() | 114 tests = TestList() |
114 tests.add('failures/expected/crash.html', crash=True) | 115 tests.add('failures/expected/crash.html', crash=True) |
115 tests.add('failures/expected/exception.html', exception=True) | 116 tests.add('failures/expected/exception.html', exception=True) |
116 tests.add('failures/expected/device_failure.html', device_failure=True) | 117 tests.add('failures/expected/device_failure.html', device_failure=True) |
117 tests.add('failures/expected/timeout.html', timeout=True) | 118 tests.add('failures/expected/timeout.html', timeout=True) |
118 tests.add('failures/expected/leak.html', leak=True) | 119 tests.add('failures/expected/leak.html', leak=True) |
119 tests.add('failures/expected/missing_text.html', expected_text=None) | 120 tests.add('failures/expected/missing_text.html', expected_text=None) |
120 tests.add('failures/expected/needsrebaseline.html', actual_text='needsrebase line text') | 121 tests.add('failures/expected/needsrebaseline.html', actual_text='needsrebase line text') |
121 tests.add('failures/expected/needsmanualrebaseline.html', actual_text='needs manualrebaseline text') | 122 tests.add('failures/expected/needsmanualrebaseline.html', actual_text='needs manualrebaseline text') |
122 tests.add('failures/expected/image.html', | 123 tests.add('failures/expected/image.html', |
123 actual_image='image_fail-pngtEXtchecksum\x00checksum_fail', | 124 actual_image='image_fail-pngtEXtchecksum\x00checksum_fail', |
124 expected_image='image-pngtEXtchecksum\x00checksum-png') | 125 expected_image='image-pngtEXtchecksum\x00checksum-png') |
125 tests.add('failures/expected/image_checksum.html', | 126 tests.add('failures/expected/image_checksum.html', |
126 actual_checksum='image_checksum_fail-checksum', | 127 actual_checksum='image_checksum_fail-checksum', |
127 actual_image='image_checksum_fail-png') | 128 actual_image='image_checksum_fail-png') |
128 tests.add('failures/expected/audio.html', | 129 tests.add('failures/expected/audio.html', |
129 actual_audio=base64.b64encode('audio_fail-wav'), expected_audio='a udio-wav', | 130 actual_audio=base64.b64encode('audio_fail-wav'), |
130 actual_text=None, expected_text=None, | 131 expected_audio='audio-wav', |
131 actual_image=None, expected_image=None, | 132 actual_text=None, |
133 expected_text=None, | |
134 actual_image=None, | |
135 expected_image=None, | |
132 actual_checksum=None) | 136 actual_checksum=None) |
133 tests.add('failures/expected/keyboard.html', keyboard=True) | 137 tests.add('failures/expected/keyboard.html', keyboard=True) |
134 tests.add('failures/expected/missing_check.html', | 138 tests.add('failures/expected/missing_check.html', expected_image='missing_ch eck-png') |
135 expected_image='missing_check-png') | |
136 tests.add('failures/expected/missing_image.html', expected_image=None) | 139 tests.add('failures/expected/missing_image.html', expected_image=None) |
137 tests.add('failures/expected/missing_audio.html', expected_audio=None, | 140 tests.add('failures/expected/missing_audio.html', |
138 actual_text=None, expected_text=None, | 141 expected_audio=None, |
139 actual_image=None, expected_image=None, | 142 actual_text=None, |
143 expected_text=None, | |
144 actual_image=None, | |
145 expected_image=None, | |
140 actual_checksum=None) | 146 actual_checksum=None) |
141 tests.add('failures/expected/missing_text.html', expected_text=None) | 147 tests.add('failures/expected/missing_text.html', expected_text=None) |
142 tests.add('failures/expected/newlines_leading.html', | 148 tests.add('failures/expected/newlines_leading.html', expected_text="\nfoo\n" , actual_text="foo\n") |
143 expected_text="\nfoo\n", actual_text="foo\n") | 149 tests.add('failures/expected/newlines_trailing.html', expected_text="foo\n\n ", actual_text="foo\n") |
144 tests.add('failures/expected/newlines_trailing.html', | 150 tests.add('failures/expected/newlines_with_excess_CR.html', expected_text="f oo\r\r\r\n", actual_text="foo\n") |
145 expected_text="foo\n\n", actual_text="foo\n") | |
146 tests.add('failures/expected/newlines_with_excess_CR.html', | |
147 expected_text="foo\r\r\r\n", actual_text="foo\n") | |
148 tests.add('failures/expected/text.html', actual_text='text_fail-png') | 151 tests.add('failures/expected/text.html', actual_text='text_fail-png') |
149 tests.add('failures/expected/crash_then_text.html') | 152 tests.add('failures/expected/crash_then_text.html') |
150 tests.add('failures/expected/skip_text.html', actual_text='text diff') | 153 tests.add('failures/expected/skip_text.html', actual_text='text diff') |
151 tests.add('failures/flaky/text.html') | 154 tests.add('failures/flaky/text.html') |
152 tests.add('failures/unexpected/missing_text.html', expected_text=None) | 155 tests.add('failures/unexpected/missing_text.html', expected_text=None) |
153 tests.add('failures/unexpected/missing_check.html', expected_image='missing- check-png') | 156 tests.add('failures/unexpected/missing_check.html', expected_image='missing- check-png') |
154 tests.add('failures/unexpected/missing_image.html', expected_image=None) | 157 tests.add('failures/unexpected/missing_image.html', expected_image=None) |
155 tests.add('failures/unexpected/missing_render_tree_dump.html', actual_text=" ""layer at (0,0) size 800x600 | 158 tests.add('failures/unexpected/missing_render_tree_dump.html', |
159 actual_text="""layer at (0,0) size 800x600 | |
156 RenderView at (0,0) size 800x600 | 160 RenderView at (0,0) size 800x600 |
157 layer at (0,0) size 800x34 | 161 layer at (0,0) size 800x34 |
158 RenderBlock {HTML} at (0,0) size 800x34 | 162 RenderBlock {HTML} at (0,0) size 800x34 |
159 RenderBody {BODY} at (8,8) size 784x18 | 163 RenderBody {BODY} at (8,8) size 784x18 |
160 RenderText {#text} at (0,0) size 133x18 | 164 RenderText {#text} at (0,0) size 133x18 |
161 text run at (0,0) width 133: "This is an image test!" | 165 text run at (0,0) width 133: "This is an image test!" |
162 """, expected_text=None) | 166 """, |
167 expected_text=None) | |
163 tests.add('failures/unexpected/crash.html', crash=True) | 168 tests.add('failures/unexpected/crash.html', crash=True) |
164 tests.add('failures/unexpected/crash-with-stderr.html', crash=True, | 169 tests.add('failures/unexpected/crash-with-stderr.html', crash=True, error="m ock-std-error-output") |
165 error="mock-std-error-output") | 170 tests.add('failures/unexpected/web-process-crash-with-stderr.html', web_proc ess_crash=True, error="mock-std-error-output") |
166 tests.add('failures/unexpected/web-process-crash-with-stderr.html', web_proc ess_crash=True, | |
167 error="mock-std-error-output") | |
168 tests.add('failures/unexpected/pass.html') | 171 tests.add('failures/unexpected/pass.html') |
169 tests.add('failures/unexpected/text-checksum.html', | 172 tests.add('failures/unexpected/text-checksum.html', |
170 actual_text='text-checksum_fail-txt', | 173 actual_text='text-checksum_fail-txt', |
171 actual_checksum='text-checksum_fail-checksum') | 174 actual_checksum='text-checksum_fail-checksum') |
172 tests.add('failures/unexpected/text-image-checksum.html', | 175 tests.add('failures/unexpected/text-image-checksum.html', |
173 actual_text='text-image-checksum_fail-txt', | 176 actual_text='text-image-checksum_fail-txt', |
174 actual_image='text-image-checksum_fail-pngtEXtchecksum\x00checksum _fail', | 177 actual_image='text-image-checksum_fail-pngtEXtchecksum\x00checksum _fail', |
175 actual_checksum='text-image-checksum_fail-checksum') | 178 actual_checksum='text-image-checksum_fail-checksum') |
176 tests.add('failures/unexpected/checksum-with-matching-image.html', | 179 tests.add('failures/unexpected/checksum-with-matching-image.html', actual_ch ecksum='text-image-checksum_fail-checksum') |
177 actual_checksum='text-image-checksum_fail-checksum') | |
178 tests.add('failures/unexpected/skip_pass.html') | 180 tests.add('failures/unexpected/skip_pass.html') |
179 tests.add('failures/unexpected/text.html', actual_text='text_fail-txt') | 181 tests.add('failures/unexpected/text.html', actual_text='text_fail-txt') |
180 tests.add('failures/unexpected/text_then_crash.html') | 182 tests.add('failures/unexpected/text_then_crash.html') |
181 tests.add('failures/unexpected/timeout.html', timeout=True) | 183 tests.add('failures/unexpected/timeout.html', timeout=True) |
182 tests.add('failures/unexpected/leak.html', leak=True) | 184 tests.add('failures/unexpected/leak.html', leak=True) |
183 tests.add('http/tests/passes/text.html') | 185 tests.add('http/tests/passes/text.html') |
184 tests.add('http/tests/passes/image.html') | 186 tests.add('http/tests/passes/image.html') |
185 tests.add('http/tests/ssl/text.html') | 187 tests.add('http/tests/ssl/text.html') |
186 tests.add('passes/args.html') | 188 tests.add('passes/args.html') |
187 tests.add('passes/error.html', error='stuff going to stderr') | 189 tests.add('passes/error.html', error='stuff going to stderr') |
188 tests.add('passes/image.html') | 190 tests.add('passes/image.html') |
189 tests.add('passes/audio.html', | 191 tests.add('passes/audio.html', |
190 actual_audio=base64.b64encode('audio-wav'), expected_audio='audio- wav', | 192 actual_audio=base64.b64encode('audio-wav'), |
191 actual_text=None, expected_text=None, | 193 expected_audio='audio-wav', |
192 actual_image=None, expected_image=None, | 194 actual_text=None, |
195 expected_text=None, | |
196 actual_image=None, | |
197 expected_image=None, | |
193 actual_checksum=None) | 198 actual_checksum=None) |
qyearsley
2016/03/18 00:33:39
Yapf also reformats arguments to functions like th
| |
194 tests.add('passes/platform_image.html') | 199 tests.add('passes/platform_image.html') |
195 tests.add('passes/checksum_in_image.html', | 200 tests.add('passes/checksum_in_image.html', expected_image='tEXtchecksum\x00c hecksum_in_image-checksum') |
196 expected_image='tEXtchecksum\x00checksum_in_image-checksum') | |
197 tests.add('passes/skipped/skip.html') | 201 tests.add('passes/skipped/skip.html') |
198 | 202 |
199 # Note that here the checksums don't match but the images do, so this test p asses "unexpectedly". | 203 # Note that here the checksums don't match but the images do, so this test p asses "unexpectedly". |
200 # See https://bugs.webkit.org/show_bug.cgi?id=69444 . | 204 # See https://bugs.webkit.org/show_bug.cgi?id=69444 . |
201 tests.add('failures/unexpected/checksum.html', actual_checksum='checksum_fai l-checksum') | 205 tests.add('failures/unexpected/checksum.html', actual_checksum='checksum_fai l-checksum') |
202 | 206 |
203 # Text output files contain "\r\n" on Windows. This may be | 207 # Text output files contain "\r\n" on Windows. This may be |
204 # helpfully filtered to "\r\r\n" by our Python/Cygwin tooling. | 208 # helpfully filtered to "\r\r\n" by our Python/Cygwin tooling. |
205 tests.add('passes/text.html', | 209 tests.add('passes/text.html', expected_text='\nfoo\n\n', actual_text='\nfoo\ r\n\r\r\n') |
206 expected_text='\nfoo\n\n', actual_text='\nfoo\r\n\r\r\n') | |
207 | 210 |
208 # For reftests. | 211 # For reftests. |
209 tests.add_reftest('passes/reftest.html', 'passes/reftest-expected.html', sam e_image=True) | 212 tests.add_reftest('passes/reftest.html', 'passes/reftest-expected.html', sam e_image=True) |
210 | 213 |
211 # This adds a different virtual reference to ensure that that also works. | 214 # This adds a different virtual reference to ensure that that also works. |
212 tests.add('virtual/virtual_passes/passes/reftest-expected.html', actual_chec ksum='xxx', actual_image='XXX', is_reftest=True) | 215 tests.add('virtual/virtual_passes/passes/reftest-expected.html', actual_chec ksum='xxx', actual_image='XXX', is_reftest=True) |
213 | 216 |
214 tests.add_reftest('passes/mismatch.html', 'passes/mismatch-expected-mismatch .html', same_image=False) | 217 tests.add_reftest('passes/mismatch.html', 'passes/mismatch-expected-mismatch .html', same_image=False) |
215 tests.add_reftest('passes/svgreftest.svg', 'passes/svgreftest-expected.svg', same_image=True) | 218 tests.add_reftest('passes/svgreftest.svg', 'passes/svgreftest-expected.svg', same_image=True) |
216 tests.add_reftest('passes/xhtreftest.xht', 'passes/xhtreftest-expected.html' , same_image=True) | 219 tests.add_reftest('passes/xhtreftest.xht', 'passes/xhtreftest-expected.html' , same_image=True) |
217 tests.add_reftest('passes/phpreftest.php', 'passes/phpreftest-expected-misma tch.svg', same_image=False) | 220 tests.add_reftest('passes/phpreftest.php', 'passes/phpreftest-expected-misma tch.svg', same_image=False) |
218 tests.add_reftest('failures/expected/reftest.html', 'failures/expected/refte st-expected.html', same_image=False) | 221 tests.add_reftest('failures/expected/reftest.html', 'failures/expected/refte st-expected.html', same_image=False) |
219 tests.add_reftest('failures/expected/mismatch.html', 'failures/expected/mism atch-expected-mismatch.html', same_image=True) | 222 tests.add_reftest('failures/expected/mismatch.html', 'failures/expected/mism atch-expected-mismatch.html', same_image=True) |
220 tests.add_reftest('failures/unexpected/crash-reftest.html', 'failures/unexpe cted/crash-reftest-expected.html', same_image=True, crash=True) | 223 tests.add_reftest('failures/unexpected/crash-reftest.html', |
224 'failures/unexpected/crash-reftest-expected.html', | |
225 same_image=True, | |
226 crash=True) | |
221 tests.add_reftest('failures/unexpected/reftest.html', 'failures/unexpected/r eftest-expected.html', same_image=False) | 227 tests.add_reftest('failures/unexpected/reftest.html', 'failures/unexpected/r eftest-expected.html', same_image=False) |
222 tests.add_reftest('failures/unexpected/mismatch.html', 'failures/unexpected/ mismatch-expected-mismatch.html', same_image=True) | 228 tests.add_reftest('failures/unexpected/mismatch.html', 'failures/unexpected/ mismatch-expected-mismatch.html', same_image=True) |
223 tests.add('failures/unexpected/reftest-nopixel.html', actual_checksum=None, actual_image=None, is_reftest=True) | 229 tests.add('failures/unexpected/reftest-nopixel.html', actual_checksum=None, actual_image=None, is_reftest=True) |
224 tests.add('failures/unexpected/reftest-nopixel-expected.html', actual_checks um=None, actual_image=None, is_reftest=True) | 230 tests.add('failures/unexpected/reftest-nopixel-expected.html', actual_checks um=None, actual_image=None, is_reftest=True) |
225 tests.add('reftests/foo/test.html') | 231 tests.add('reftests/foo/test.html') |
226 tests.add('reftests/foo/test-ref.html') | 232 tests.add('reftests/foo/test-ref.html') |
227 | 233 |
228 tests.add('reftests/foo/multiple-match-success.html', actual_checksum='abc', actual_image='abc') | 234 tests.add('reftests/foo/multiple-match-success.html', actual_checksum='abc', actual_image='abc') |
229 tests.add('reftests/foo/multiple-match-failure.html', actual_checksum='abc', actual_image='abc') | 235 tests.add('reftests/foo/multiple-match-failure.html', actual_checksum='abc', actual_image='abc') |
230 tests.add('reftests/foo/multiple-mismatch-success.html', actual_checksum='ab c', actual_image='abc') | 236 tests.add('reftests/foo/multiple-mismatch-success.html', actual_checksum='ab c', actual_image='abc') |
(...skipping 15 matching lines...) Expand all Loading... | |
246 # For testing that we don't run tests under platform/. Note that these don't contribute to TOTAL_TESTS. | 252 # For testing that we don't run tests under platform/. Note that these don't contribute to TOTAL_TESTS. |
247 tests.add('platform/test-mac-10.10/http/test.html') | 253 tests.add('platform/test-mac-10.10/http/test.html') |
248 tests.add('platform/test-win-win7/http/test.html') | 254 tests.add('platform/test-win-win7/http/test.html') |
249 | 255 |
250 # For testing if perf tests are running in a locked shard. | 256 # For testing if perf tests are running in a locked shard. |
251 tests.add('perf/foo/test.html') | 257 tests.add('perf/foo/test.html') |
252 tests.add('perf/foo/test-ref.html') | 258 tests.add('perf/foo/test-ref.html') |
253 | 259 |
254 # For testing --pixel-test-directories. | 260 # For testing --pixel-test-directories. |
255 tests.add('failures/unexpected/pixeldir/image_in_pixeldir.html', | 261 tests.add('failures/unexpected/pixeldir/image_in_pixeldir.html', |
256 actual_image='image_in_pixeldir-pngtEXtchecksum\x00checksum_fail', | 262 actual_image='image_in_pixeldir-pngtEXtchecksum\x00checksum_fail', |
257 expected_image='image_in_pixeldir-pngtEXtchecksum\x00checksum-png') | 263 expected_image='image_in_pixeldir-pngtEXtchecksum\x00checksum-png' ) |
258 tests.add('failures/unexpected/image_not_in_pixeldir.html', | 264 tests.add('failures/unexpected/image_not_in_pixeldir.html', |
259 actual_image='image_not_in_pixeldir-pngtEXtchecksum\x00checksum_fail', | 265 actual_image='image_not_in_pixeldir-pngtEXtchecksum\x00checksum_fa il', |
260 expected_image='image_not_in_pixeldir-pngtEXtchecksum\x00checksum-png') | 266 expected_image='image_not_in_pixeldir-pngtEXtchecksum\x00checksum- png') |
261 | 267 |
262 # For testing that virtual test suites don't expand names containing themsel ves | 268 # For testing that virtual test suites don't expand names containing themsel ves |
263 # See webkit.org/b/97925 and base_unittest.PortTest.test_tests(). | 269 # See webkit.org/b/97925 and base_unittest.PortTest.test_tests(). |
264 tests.add('passes/test-virtual-passes.html') | 270 tests.add('passes/test-virtual-passes.html') |
265 tests.add('passes/virtual_passes/test-virtual-passes.html') | 271 tests.add('passes/virtual_passes/test-virtual-passes.html') |
266 | 272 |
267 return tests | 273 return tests |
268 | 274 |
269 | |
270 # Here we use a non-standard location for the layout tests, to ensure that | 275 # Here we use a non-standard location for the layout tests, to ensure that |
271 # this works. The path contains a '.' in the name because we've seen bugs | 276 # this works. The path contains a '.' in the name because we've seen bugs |
272 # related to this before. | 277 # related to this before. |
273 | 278 |
274 LAYOUT_TEST_DIR = '/test.checkout/LayoutTests' | 279 LAYOUT_TEST_DIR = '/test.checkout/LayoutTests' |
275 PERF_TEST_DIR = '/test.checkout/PerformanceTests' | 280 PERF_TEST_DIR = '/test.checkout/PerformanceTests' |
276 | 281 |
277 | 282 |
278 # Here we synthesize an in-memory filesystem from the test list | 283 # Here we synthesize an in-memory filesystem from the test list |
279 # in order to fully control the test output and to demonstrate that | 284 # in order to fully control the test output and to demonstrate that |
(...skipping 66 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
346 for test in test_list.tests.values(): | 351 for test in test_list.tests.values(): |
347 add_file(test, test.name[test.name.rfind('.'):], '') | 352 add_file(test, test.name[test.name.rfind('.'):], '') |
348 if test.is_reftest: | 353 if test.is_reftest: |
349 continue | 354 continue |
350 if test.actual_audio: | 355 if test.actual_audio: |
351 add_file(test, '-expected.wav', test.expected_audio) | 356 add_file(test, '-expected.wav', test.expected_audio) |
352 continue | 357 continue |
353 add_file(test, '-expected.txt', test.expected_text) | 358 add_file(test, '-expected.txt', test.expected_text) |
354 add_file(test, '-expected.png', test.expected_image) | 359 add_file(test, '-expected.png', test.expected_image) |
355 | 360 |
356 filesystem.write_text_file(filesystem.join(LAYOUT_TEST_DIR, 'virtual', 'virt ual_passes', 'passes', 'args-expected.txt'), 'args-txt --virtual-arg') | 361 filesystem.write_text_file( |
362 filesystem.join(LAYOUT_TEST_DIR, 'virtual', 'virtual_passes', 'passes', 'args-expected.txt'), 'args-txt --virtual-arg') | |
357 # Clear the list of written files so that we can watch what happens during t esting. | 363 # Clear the list of written files so that we can watch what happens during t esting. |
358 filesystem.clear_written_files() | 364 filesystem.clear_written_files() |
359 | 365 |
360 | 366 |
361 class TestPort(Port): | 367 class TestPort(Port): |
362 port_name = 'test' | 368 port_name = 'test' |
363 default_port_name = 'test-mac-mac10.10' | 369 default_port_name = 'test-mac-mac10.10' |
364 | 370 |
365 # TODO(wkorman): The below constant is legacy code and is only referenced by a unit test. Find the modern way to do | 371 # TODO(wkorman): The below constant is legacy code and is only referenced by a unit test. Find the modern way to do |
366 # the same thing that test is doing and delete this. | 372 # the same thing that test is doing and delete this. |
367 # | 373 # |
368 # A list of platform names sufficient to cover all the baselines. | 374 # A list of platform names sufficient to cover all the baselines. |
369 # The list should be sorted so that a later platform will reuse | 375 # The list should be sorted so that a later platform will reuse |
370 # an earlier platform's baselines if they are the same (e.g., | 376 # an earlier platform's baselines if they are the same (e.g., |
371 # 'mac10.10' should precede 'mac10.9'). | 377 # 'mac10.10' should precede 'mac10.9'). |
372 ALL_BASELINE_VARIANTS = ( | 378 ALL_BASELINE_VARIANTS = ('test-linux-trusty', 'test-linux-precise', 'test-li nux-x86', 'test-mac-mac10.11', 'test-mac-mac10.10', |
373 'test-linux-trusty', 'test-linux-precise', 'test-linux-x86', | 379 'test-win-win10', 'test-win-win7') |
374 'test-mac-mac10.11', 'test-mac-mac10.10', | |
375 'test-win-win10', 'test-win-win7' | |
376 ) | |
377 | 380 |
378 FALLBACK_PATHS = { | 381 FALLBACK_PATHS = { |
379 'win7': ['test-win-win7', 'test-win-win10'], | 382 'win7': ['test-win-win7', 'test-win-win10'], |
380 'win10': ['test-win-win10'], | 383 'win10': ['test-win-win10'], |
381 'mac10.10': ['test-mac-mac10.10', 'test-mac-mac10.11'], | 384 'mac10.10': ['test-mac-mac10.10', 'test-mac-mac10.11'], |
382 'mac10.11': ['test-mac-mac10.11'], | 385 'mac10.11': ['test-mac-mac10.11'], |
383 'trusty': ['test-linux-trusty', 'test-win-win7'], | 386 'trusty': ['test-linux-trusty', 'test-win-win7'], |
384 'precise': ['test-linux-precise', 'test-linux-trusty', 'test-win-win 7'], | 387 'precise': ['test-linux-precise', 'test-linux-trusty', 'test-win-win7'], |
385 'linux32': ['test-linux-x86', 'test-linux-precise', 'test-linux-trus ty', 'test-win-win7'], | 388 'linux32': ['test-linux-x86', 'test-linux-precise', 'test-linux-trusty', 'test-win-win7'], |
386 } | 389 } |
387 | 390 |
388 @classmethod | 391 @classmethod |
389 def determine_full_port_name(cls, host, options, port_name): | 392 def determine_full_port_name(cls, host, options, port_name): |
390 if port_name == 'test': | 393 if port_name == 'test': |
391 return TestPort.default_port_name | 394 return TestPort.default_port_name |
392 return port_name | 395 return port_name |
393 | 396 |
394 def __init__(self, host, port_name=None, **kwargs): | 397 def __init__(self, host, port_name=None, **kwargs): |
395 Port.__init__(self, host, port_name or TestPort.default_port_name, **kwa rgs) | 398 Port.__init__(self, host, port_name or TestPort.default_port_name, **kwa rgs) |
(...skipping 68 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
464 def layout_tests_dir(self): | 467 def layout_tests_dir(self): |
465 return LAYOUT_TEST_DIR | 468 return LAYOUT_TEST_DIR |
466 | 469 |
467 def perf_tests_dir(self): | 470 def perf_tests_dir(self): |
468 return PERF_TEST_DIR | 471 return PERF_TEST_DIR |
469 | 472 |
470 def webkit_base(self): | 473 def webkit_base(self): |
471 return '/test.checkout' | 474 return '/test.checkout' |
472 | 475 |
473 def _skipped_tests_for_unsupported_features(self, test_list): | 476 def _skipped_tests_for_unsupported_features(self, test_list): |
474 return set(['failures/expected/skip_text.html', | 477 return set(['failures/expected/skip_text.html', 'failures/unexpected/ski p_pass.html', 'virtual/skipped/failures/expected']) |
475 'failures/unexpected/skip_pass.html', | |
476 'virtual/skipped/failures/expected']) | |
477 | 478 |
478 def name(self): | 479 def name(self): |
479 return self._name | 480 return self._name |
480 | 481 |
481 def operating_system(self): | 482 def operating_system(self): |
482 return self._operating_system | 483 return self._operating_system |
483 | 484 |
484 def _path_to_wdiff(self): | 485 def _path_to_wdiff(self): |
485 return None | 486 return None |
486 | 487 |
(...skipping 36 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
523 def _port_specific_expectations_files(self): | 524 def _port_specific_expectations_files(self): |
524 return [self._filesystem.join(self._webkit_baseline_path(d), 'TestExpect ations') for d in ['test', 'test-win-win7']] | 525 return [self._filesystem.join(self._webkit_baseline_path(d), 'TestExpect ations') for d in ['test', 'test-win-win7']] |
525 | 526 |
526 def all_test_configurations(self): | 527 def all_test_configurations(self): |
527 """Returns a sequence of the TestConfigurations the port supports.""" | 528 """Returns a sequence of the TestConfigurations the port supports.""" |
528 # By default, we assume we want to test every graphics type in | 529 # By default, we assume we want to test every graphics type in |
529 # every configuration on every system. | 530 # every configuration on every system. |
530 test_configurations = [] | 531 test_configurations = [] |
531 for version, architecture in self._all_systems(): | 532 for version, architecture in self._all_systems(): |
532 for build_type in self._all_build_types(): | 533 for build_type in self._all_build_types(): |
533 test_configurations.append(TestConfiguration( | 534 test_configurations.append(TestConfiguration(version=version, ar chitecture=architecture, build_type=build_type)) |
534 version=version, | |
535 architecture=architecture, | |
536 build_type=build_type)) | |
537 return test_configurations | 535 return test_configurations |
538 | 536 |
539 def _all_systems(self): | 537 def _all_systems(self): |
540 return (('mac10.10', 'x86'), | 538 return (('mac10.10', 'x86'), ('mac10.11', 'x86'), ('win7', 'x86'), ('win 10', 'x86'), ('linux32', 'x86'), |
541 ('mac10.11', 'x86'), | 539 ('precise', 'x86_64'), ('trusty', 'x86_64')) |
542 ('win7', 'x86'), | |
543 ('win10', 'x86'), | |
544 ('linux32', 'x86'), | |
545 ('precise', 'x86_64'), | |
546 ('trusty', 'x86_64')) | |
547 | 540 |
548 def _all_build_types(self): | 541 def _all_build_types(self): |
549 return ('debug', 'release') | 542 return ('debug', 'release') |
550 | 543 |
551 def configuration_specifier_macros(self): | 544 def configuration_specifier_macros(self): |
552 """To avoid surprises when introducing new macros, these are intentional ly fixed in time.""" | 545 """To avoid surprises when introducing new macros, these are intentional ly fixed in time.""" |
553 return { | 546 return {'mac': ['mac10.10', 'mac10.11'], 'win': ['win7', 'win10'], 'linu x': ['linux32', 'precise', 'trusty']} |
554 'mac': ['mac10.10', 'mac10.11'], | |
555 'win': ['win7', 'win10'], | |
556 'linux': ['linux32', 'precise', 'trusty'] | |
557 } | |
558 | 547 |
559 def virtual_test_suites(self): | 548 def virtual_test_suites(self): |
560 return [ | 549 return [ |
561 VirtualTestSuite(prefix='virtual_passes', base='passes', args=['--vi rtual-arg']), | 550 VirtualTestSuite(prefix='virtual_passes', |
562 VirtualTestSuite(prefix='skipped', base='failures/expected', args=[' --virtual-arg2']), | 551 base='passes', |
563 VirtualTestSuite(prefix='references_use_default_args', base='passes/ reftest.html', | 552 args=['--virtual-arg']), |
564 args=['--virtual-arg'], references_use_default_args =True), | 553 VirtualTestSuite(prefix='skipped', |
554 base='failures/expected', | |
555 args=['--virtual-arg2']), | |
556 VirtualTestSuite(prefix='references_use_default_args', | |
557 base='passes/reftest.html', | |
558 args=['--virtual-arg'], | |
559 references_use_default_args=True), | |
565 ] | 560 ] |
566 | 561 |
567 | 562 |
568 class TestDriver(Driver): | 563 class TestDriver(Driver): |
569 """Test/Dummy implementation of the driver interface.""" | 564 """Test/Dummy implementation of the driver interface.""" |
570 next_pid = 1 | 565 next_pid = 1 |
571 | 566 |
572 def __init__(self, *args, **kwargs): | 567 def __init__(self, *args, **kwargs): |
573 super(TestDriver, self).__init__(*args, **kwargs) | 568 super(TestDriver, self).__init__(*args, **kwargs) |
574 self.started = False | 569 self.started = False |
575 self.pid = 0 | 570 self.pid = 0 |
576 | 571 |
577 def cmd_line(self, pixel_tests, per_test_args): | 572 def cmd_line(self, pixel_tests, per_test_args): |
578 pixel_tests_flag = '-p' if pixel_tests else '' | 573 pixel_tests_flag = '-p' if pixel_tests else '' |
579 return [self._port._path_to_driver()] + [pixel_tests_flag] + self._port. get_option('additional_driver_flag', []) + per_test_args | 574 return [self._port._path_to_driver()] + [pixel_tests_flag] + self._port. get_option('additional_driver_flag', |
575 []) + per_test_args | |
580 | 576 |
581 def run_test(self, driver_input, stop_when_done): | 577 def run_test(self, driver_input, stop_when_done): |
582 if not self.started: | 578 if not self.started: |
583 self.started = True | 579 self.started = True |
584 self.pid = TestDriver.next_pid | 580 self.pid = TestDriver.next_pid |
585 TestDriver.next_pid += 1 | 581 TestDriver.next_pid += 1 |
586 | 582 |
587 start_time = time.time() | 583 start_time = time.time() |
588 test_name = driver_input.test_name | 584 test_name = driver_input.test_name |
589 test_args = driver_input.args or [] | 585 test_args = driver_input.args or [] |
(...skipping 57 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
647 crash = True | 643 crash = True |
648 crash_log = 'reftest crash log' | 644 crash_log = 'reftest crash log' |
649 | 645 |
650 if stop_when_done: | 646 if stop_when_done: |
651 self.stop() | 647 self.stop() |
652 | 648 |
653 if test.actual_checksum == driver_input.image_hash: | 649 if test.actual_checksum == driver_input.image_hash: |
654 image = None | 650 image = None |
655 else: | 651 else: |
656 image = test.actual_image | 652 image = test.actual_image |
657 return DriverOutput(actual_text, image, test.actual_checksum, audio, | 653 return DriverOutput(actual_text, |
658 crash=(crash or web_process_crash), crashed_process_name=crashed_pro cess_name, | 654 image, |
659 crashed_pid=crashed_pid, crash_log=crash_log, | 655 test.actual_checksum, |
660 test_time=time.time() - start_time, timeout=test.timeout, error=test .error, pid=self.pid, | 656 audio, |
661 leak=test.leak) | 657 crash=(crash or web_process_crash), |
658 crashed_process_name=crashed_process_name, | |
659 crashed_pid=crashed_pid, | |
660 crash_log=crash_log, | |
661 test_time=time.time() - start_time, | |
662 timeout=test.timeout, | |
663 error=test.error, | |
664 pid=self.pid, | |
665 leak=test.leak) | |
662 | 666 |
663 def stop(self): | 667 def stop(self): |
664 self.started = False | 668 self.started = False |
OLD | NEW |