| OLD | NEW |
| 1 # Copyright (C) 2010 Google Inc. All rights reserved. | 1 # Copyright (C) 2010 Google Inc. All rights reserved. |
| 2 # | 2 # |
| 3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
| 4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
| 5 # met: | 5 # met: |
| 6 # | 6 # |
| 7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
| 8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
| 9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
| 10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
| (...skipping 91 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 102 | 102 |
| 103 # | 103 # |
| 104 # These numbers may need to be updated whenever we add or delete tests. This inc
ludes virtual tests. | 104 # These numbers may need to be updated whenever we add or delete tests. This inc
ludes virtual tests. |
| 105 # | 105 # |
| 106 TOTAL_TESTS = 114 | 106 TOTAL_TESTS = 114 |
| 107 TOTAL_SKIPS = 26 | 107 TOTAL_SKIPS = 26 |
| 108 TOTAL_CRASHES = 80 | 108 TOTAL_CRASHES = 80 |
| 109 UNEXPECTED_PASSES = 1 | 109 UNEXPECTED_PASSES = 1 |
| 110 UNEXPECTED_FAILURES = 26 | 110 UNEXPECTED_FAILURES = 26 |
| 111 | 111 |
| 112 |
| 112 def unit_test_list(): | 113 def unit_test_list(): |
| 113 tests = TestList() | 114 tests = TestList() |
| 114 tests.add('failures/expected/crash.html', crash=True) | 115 tests.add('failures/expected/crash.html', crash=True) |
| 115 tests.add('failures/expected/exception.html', exception=True) | 116 tests.add('failures/expected/exception.html', exception=True) |
| 116 tests.add('failures/expected/device_failure.html', device_failure=True) | 117 tests.add('failures/expected/device_failure.html', device_failure=True) |
| 117 tests.add('failures/expected/timeout.html', timeout=True) | 118 tests.add('failures/expected/timeout.html', timeout=True) |
| 118 tests.add('failures/expected/leak.html', leak=True) | 119 tests.add('failures/expected/leak.html', leak=True) |
| 119 tests.add('failures/expected/missing_text.html', expected_text=None) | 120 tests.add('failures/expected/missing_text.html', expected_text=None) |
| 120 tests.add('failures/expected/needsrebaseline.html', actual_text='needsrebase
line text') | 121 tests.add('failures/expected/needsrebaseline.html', actual_text='needsrebase
line text') |
| 121 tests.add('failures/expected/needsmanualrebaseline.html', actual_text='needs
manualrebaseline text') | 122 tests.add('failures/expected/needsmanualrebaseline.html', actual_text='needs
manualrebaseline text') |
| 122 tests.add('failures/expected/image.html', | 123 tests.add('failures/expected/image.html', |
| 123 actual_image='image_fail-pngtEXtchecksum\x00checksum_fail', | 124 actual_image='image_fail-pngtEXtchecksum\x00checksum_fail', |
| 124 expected_image='image-pngtEXtchecksum\x00checksum-png') | 125 expected_image='image-pngtEXtchecksum\x00checksum-png') |
| 125 tests.add('failures/expected/image_checksum.html', | 126 tests.add('failures/expected/image_checksum.html', |
| 126 actual_checksum='image_checksum_fail-checksum', | 127 actual_checksum='image_checksum_fail-checksum', |
| 127 actual_image='image_checksum_fail-png') | 128 actual_image='image_checksum_fail-png') |
| 128 tests.add('failures/expected/audio.html', | 129 tests.add('failures/expected/audio.html', |
| 129 actual_audio=base64.b64encode('audio_fail-wav'), expected_audio='a
udio-wav', | 130 actual_audio=base64.b64encode('audio_fail-wav'), |
| 130 actual_text=None, expected_text=None, | 131 expected_audio='audio-wav', |
| 131 actual_image=None, expected_image=None, | 132 actual_text=None, |
| 133 expected_text=None, |
| 134 actual_image=None, |
| 135 expected_image=None, |
| 132 actual_checksum=None) | 136 actual_checksum=None) |
| 133 tests.add('failures/expected/keyboard.html', keyboard=True) | 137 tests.add('failures/expected/keyboard.html', keyboard=True) |
| 134 tests.add('failures/expected/missing_check.html', | 138 tests.add('failures/expected/missing_check.html', expected_image='missing_ch
eck-png') |
| 135 expected_image='missing_check-png') | |
| 136 tests.add('failures/expected/missing_image.html', expected_image=None) | 139 tests.add('failures/expected/missing_image.html', expected_image=None) |
| 137 tests.add('failures/expected/missing_audio.html', expected_audio=None, | 140 tests.add('failures/expected/missing_audio.html', |
| 138 actual_text=None, expected_text=None, | 141 expected_audio=None, |
| 139 actual_image=None, expected_image=None, | 142 actual_text=None, |
| 143 expected_text=None, |
| 144 actual_image=None, |
| 145 expected_image=None, |
| 140 actual_checksum=None) | 146 actual_checksum=None) |
| 141 tests.add('failures/expected/missing_text.html', expected_text=None) | 147 tests.add('failures/expected/missing_text.html', expected_text=None) |
| 142 tests.add('failures/expected/newlines_leading.html', | 148 tests.add('failures/expected/newlines_leading.html', expected_text="\nfoo\n"
, actual_text="foo\n") |
| 143 expected_text="\nfoo\n", actual_text="foo\n") | 149 tests.add('failures/expected/newlines_trailing.html', expected_text="foo\n\n
", actual_text="foo\n") |
| 144 tests.add('failures/expected/newlines_trailing.html', | 150 tests.add('failures/expected/newlines_with_excess_CR.html', expected_text="f
oo\r\r\r\n", actual_text="foo\n") |
| 145 expected_text="foo\n\n", actual_text="foo\n") | |
| 146 tests.add('failures/expected/newlines_with_excess_CR.html', | |
| 147 expected_text="foo\r\r\r\n", actual_text="foo\n") | |
| 148 tests.add('failures/expected/text.html', actual_text='text_fail-png') | 151 tests.add('failures/expected/text.html', actual_text='text_fail-png') |
| 149 tests.add('failures/expected/crash_then_text.html') | 152 tests.add('failures/expected/crash_then_text.html') |
| 150 tests.add('failures/expected/skip_text.html', actual_text='text diff') | 153 tests.add('failures/expected/skip_text.html', actual_text='text diff') |
| 151 tests.add('failures/flaky/text.html') | 154 tests.add('failures/flaky/text.html') |
| 152 tests.add('failures/unexpected/missing_text.html', expected_text=None) | 155 tests.add('failures/unexpected/missing_text.html', expected_text=None) |
| 153 tests.add('failures/unexpected/missing_check.html', expected_image='missing-
check-png') | 156 tests.add('failures/unexpected/missing_check.html', expected_image='missing-
check-png') |
| 154 tests.add('failures/unexpected/missing_image.html', expected_image=None) | 157 tests.add('failures/unexpected/missing_image.html', expected_image=None) |
| 155 tests.add('failures/unexpected/missing_render_tree_dump.html', actual_text="
""layer at (0,0) size 800x600 | 158 tests.add('failures/unexpected/missing_render_tree_dump.html', |
| 159 actual_text="""layer at (0,0) size 800x600 |
| 156 RenderView at (0,0) size 800x600 | 160 RenderView at (0,0) size 800x600 |
| 157 layer at (0,0) size 800x34 | 161 layer at (0,0) size 800x34 |
| 158 RenderBlock {HTML} at (0,0) size 800x34 | 162 RenderBlock {HTML} at (0,0) size 800x34 |
| 159 RenderBody {BODY} at (8,8) size 784x18 | 163 RenderBody {BODY} at (8,8) size 784x18 |
| 160 RenderText {#text} at (0,0) size 133x18 | 164 RenderText {#text} at (0,0) size 133x18 |
| 161 text run at (0,0) width 133: "This is an image test!" | 165 text run at (0,0) width 133: "This is an image test!" |
| 162 """, expected_text=None) | 166 """, |
| 167 expected_text=None) |
| 163 tests.add('failures/unexpected/crash.html', crash=True) | 168 tests.add('failures/unexpected/crash.html', crash=True) |
| 164 tests.add('failures/unexpected/crash-with-stderr.html', crash=True, | 169 tests.add('failures/unexpected/crash-with-stderr.html', crash=True, error="m
ock-std-error-output") |
| 165 error="mock-std-error-output") | 170 tests.add('failures/unexpected/web-process-crash-with-stderr.html', web_proc
ess_crash=True, error="mock-std-error-output") |
| 166 tests.add('failures/unexpected/web-process-crash-with-stderr.html', web_proc
ess_crash=True, | |
| 167 error="mock-std-error-output") | |
| 168 tests.add('failures/unexpected/pass.html') | 171 tests.add('failures/unexpected/pass.html') |
| 169 tests.add('failures/unexpected/text-checksum.html', | 172 tests.add('failures/unexpected/text-checksum.html', |
| 170 actual_text='text-checksum_fail-txt', | 173 actual_text='text-checksum_fail-txt', |
| 171 actual_checksum='text-checksum_fail-checksum') | 174 actual_checksum='text-checksum_fail-checksum') |
| 172 tests.add('failures/unexpected/text-image-checksum.html', | 175 tests.add('failures/unexpected/text-image-checksum.html', |
| 173 actual_text='text-image-checksum_fail-txt', | 176 actual_text='text-image-checksum_fail-txt', |
| 174 actual_image='text-image-checksum_fail-pngtEXtchecksum\x00checksum
_fail', | 177 actual_image='text-image-checksum_fail-pngtEXtchecksum\x00checksum
_fail', |
| 175 actual_checksum='text-image-checksum_fail-checksum') | 178 actual_checksum='text-image-checksum_fail-checksum') |
| 176 tests.add('failures/unexpected/checksum-with-matching-image.html', | 179 tests.add('failures/unexpected/checksum-with-matching-image.html', actual_ch
ecksum='text-image-checksum_fail-checksum') |
| 177 actual_checksum='text-image-checksum_fail-checksum') | |
| 178 tests.add('failures/unexpected/skip_pass.html') | 180 tests.add('failures/unexpected/skip_pass.html') |
| 179 tests.add('failures/unexpected/text.html', actual_text='text_fail-txt') | 181 tests.add('failures/unexpected/text.html', actual_text='text_fail-txt') |
| 180 tests.add('failures/unexpected/text_then_crash.html') | 182 tests.add('failures/unexpected/text_then_crash.html') |
| 181 tests.add('failures/unexpected/timeout.html', timeout=True) | 183 tests.add('failures/unexpected/timeout.html', timeout=True) |
| 182 tests.add('failures/unexpected/leak.html', leak=True) | 184 tests.add('failures/unexpected/leak.html', leak=True) |
| 183 tests.add('http/tests/passes/text.html') | 185 tests.add('http/tests/passes/text.html') |
| 184 tests.add('http/tests/passes/image.html') | 186 tests.add('http/tests/passes/image.html') |
| 185 tests.add('http/tests/ssl/text.html') | 187 tests.add('http/tests/ssl/text.html') |
| 186 tests.add('passes/args.html') | 188 tests.add('passes/args.html') |
| 187 tests.add('passes/error.html', error='stuff going to stderr') | 189 tests.add('passes/error.html', error='stuff going to stderr') |
| 188 tests.add('passes/image.html') | 190 tests.add('passes/image.html') |
| 189 tests.add('passes/audio.html', | 191 tests.add('passes/audio.html', |
| 190 actual_audio=base64.b64encode('audio-wav'), expected_audio='audio-
wav', | 192 actual_audio=base64.b64encode('audio-wav'), |
| 191 actual_text=None, expected_text=None, | 193 expected_audio='audio-wav', |
| 192 actual_image=None, expected_image=None, | 194 actual_text=None, |
| 195 expected_text=None, |
| 196 actual_image=None, |
| 197 expected_image=None, |
| 193 actual_checksum=None) | 198 actual_checksum=None) |
| 194 tests.add('passes/platform_image.html') | 199 tests.add('passes/platform_image.html') |
| 195 tests.add('passes/checksum_in_image.html', | 200 tests.add('passes/checksum_in_image.html', expected_image='tEXtchecksum\x00c
hecksum_in_image-checksum') |
| 196 expected_image='tEXtchecksum\x00checksum_in_image-checksum') | |
| 197 tests.add('passes/skipped/skip.html') | 201 tests.add('passes/skipped/skip.html') |
| 198 | 202 |
| 199 # Note that here the checksums don't match but the images do, so this test p
asses "unexpectedly". | 203 # Note that here the checksums don't match but the images do, so this test p
asses "unexpectedly". |
| 200 # See https://bugs.webkit.org/show_bug.cgi?id=69444 . | 204 # See https://bugs.webkit.org/show_bug.cgi?id=69444 . |
| 201 tests.add('failures/unexpected/checksum.html', actual_checksum='checksum_fai
l-checksum') | 205 tests.add('failures/unexpected/checksum.html', actual_checksum='checksum_fai
l-checksum') |
| 202 | 206 |
| 203 # Text output files contain "\r\n" on Windows. This may be | 207 # Text output files contain "\r\n" on Windows. This may be |
| 204 # helpfully filtered to "\r\r\n" by our Python/Cygwin tooling. | 208 # helpfully filtered to "\r\r\n" by our Python/Cygwin tooling. |
| 205 tests.add('passes/text.html', | 209 tests.add('passes/text.html', expected_text='\nfoo\n\n', actual_text='\nfoo\
r\n\r\r\n') |
| 206 expected_text='\nfoo\n\n', actual_text='\nfoo\r\n\r\r\n') | |
| 207 | 210 |
| 208 # For reftests. | 211 # For reftests. |
| 209 tests.add_reftest('passes/reftest.html', 'passes/reftest-expected.html', sam
e_image=True) | 212 tests.add_reftest('passes/reftest.html', 'passes/reftest-expected.html', sam
e_image=True) |
| 210 | 213 |
| 211 # This adds a different virtual reference to ensure that that also works. | 214 # This adds a different virtual reference to ensure that that also works. |
| 212 tests.add('virtual/virtual_passes/passes/reftest-expected.html', actual_chec
ksum='xxx', actual_image='XXX', is_reftest=True) | 215 tests.add('virtual/virtual_passes/passes/reftest-expected.html', actual_chec
ksum='xxx', actual_image='XXX', is_reftest=True) |
| 213 | 216 |
| 214 tests.add_reftest('passes/mismatch.html', 'passes/mismatch-expected-mismatch
.html', same_image=False) | 217 tests.add_reftest('passes/mismatch.html', 'passes/mismatch-expected-mismatch
.html', same_image=False) |
| 215 tests.add_reftest('passes/svgreftest.svg', 'passes/svgreftest-expected.svg',
same_image=True) | 218 tests.add_reftest('passes/svgreftest.svg', 'passes/svgreftest-expected.svg',
same_image=True) |
| 216 tests.add_reftest('passes/xhtreftest.xht', 'passes/xhtreftest-expected.html'
, same_image=True) | 219 tests.add_reftest('passes/xhtreftest.xht', 'passes/xhtreftest-expected.html'
, same_image=True) |
| 217 tests.add_reftest('passes/phpreftest.php', 'passes/phpreftest-expected-misma
tch.svg', same_image=False) | 220 tests.add_reftest('passes/phpreftest.php', 'passes/phpreftest-expected-misma
tch.svg', same_image=False) |
| 218 tests.add_reftest('failures/expected/reftest.html', 'failures/expected/refte
st-expected.html', same_image=False) | 221 tests.add_reftest('failures/expected/reftest.html', 'failures/expected/refte
st-expected.html', same_image=False) |
| 219 tests.add_reftest('failures/expected/mismatch.html', 'failures/expected/mism
atch-expected-mismatch.html', same_image=True) | 222 tests.add_reftest('failures/expected/mismatch.html', 'failures/expected/mism
atch-expected-mismatch.html', same_image=True) |
| 220 tests.add_reftest('failures/unexpected/crash-reftest.html', 'failures/unexpe
cted/crash-reftest-expected.html', same_image=True, crash=True) | 223 tests.add_reftest('failures/unexpected/crash-reftest.html', |
| 224 'failures/unexpected/crash-reftest-expected.html', |
| 225 same_image=True, |
| 226 crash=True) |
| 221 tests.add_reftest('failures/unexpected/reftest.html', 'failures/unexpected/r
eftest-expected.html', same_image=False) | 227 tests.add_reftest('failures/unexpected/reftest.html', 'failures/unexpected/r
eftest-expected.html', same_image=False) |
| 222 tests.add_reftest('failures/unexpected/mismatch.html', 'failures/unexpected/
mismatch-expected-mismatch.html', same_image=True) | 228 tests.add_reftest('failures/unexpected/mismatch.html', 'failures/unexpected/
mismatch-expected-mismatch.html', same_image=True) |
| 223 tests.add('failures/unexpected/reftest-nopixel.html', actual_checksum=None,
actual_image=None, is_reftest=True) | 229 tests.add('failures/unexpected/reftest-nopixel.html', actual_checksum=None,
actual_image=None, is_reftest=True) |
| 224 tests.add('failures/unexpected/reftest-nopixel-expected.html', actual_checks
um=None, actual_image=None, is_reftest=True) | 230 tests.add('failures/unexpected/reftest-nopixel-expected.html', actual_checks
um=None, actual_image=None, is_reftest=True) |
| 225 tests.add('reftests/foo/test.html') | 231 tests.add('reftests/foo/test.html') |
| 226 tests.add('reftests/foo/test-ref.html') | 232 tests.add('reftests/foo/test-ref.html') |
| 227 | 233 |
| 228 tests.add('reftests/foo/multiple-match-success.html', actual_checksum='abc',
actual_image='abc') | 234 tests.add('reftests/foo/multiple-match-success.html', actual_checksum='abc',
actual_image='abc') |
| 229 tests.add('reftests/foo/multiple-match-failure.html', actual_checksum='abc',
actual_image='abc') | 235 tests.add('reftests/foo/multiple-match-failure.html', actual_checksum='abc',
actual_image='abc') |
| 230 tests.add('reftests/foo/multiple-mismatch-success.html', actual_checksum='ab
c', actual_image='abc') | 236 tests.add('reftests/foo/multiple-mismatch-success.html', actual_checksum='ab
c', actual_image='abc') |
| (...skipping 15 matching lines...) Expand all Loading... |
| 246 # For testing that we don't run tests under platform/. Note that these don't
contribute to TOTAL_TESTS. | 252 # For testing that we don't run tests under platform/. Note that these don't
contribute to TOTAL_TESTS. |
| 247 tests.add('platform/test-mac-10.10/http/test.html') | 253 tests.add('platform/test-mac-10.10/http/test.html') |
| 248 tests.add('platform/test-win-win7/http/test.html') | 254 tests.add('platform/test-win-win7/http/test.html') |
| 249 | 255 |
| 250 # For testing if perf tests are running in a locked shard. | 256 # For testing if perf tests are running in a locked shard. |
| 251 tests.add('perf/foo/test.html') | 257 tests.add('perf/foo/test.html') |
| 252 tests.add('perf/foo/test-ref.html') | 258 tests.add('perf/foo/test-ref.html') |
| 253 | 259 |
| 254 # For testing --pixel-test-directories. | 260 # For testing --pixel-test-directories. |
| 255 tests.add('failures/unexpected/pixeldir/image_in_pixeldir.html', | 261 tests.add('failures/unexpected/pixeldir/image_in_pixeldir.html', |
| 256 actual_image='image_in_pixeldir-pngtEXtchecksum\x00checksum_fail', | 262 actual_image='image_in_pixeldir-pngtEXtchecksum\x00checksum_fail', |
| 257 expected_image='image_in_pixeldir-pngtEXtchecksum\x00checksum-png') | 263 expected_image='image_in_pixeldir-pngtEXtchecksum\x00checksum-png'
) |
| 258 tests.add('failures/unexpected/image_not_in_pixeldir.html', | 264 tests.add('failures/unexpected/image_not_in_pixeldir.html', |
| 259 actual_image='image_not_in_pixeldir-pngtEXtchecksum\x00checksum_fail', | 265 actual_image='image_not_in_pixeldir-pngtEXtchecksum\x00checksum_fa
il', |
| 260 expected_image='image_not_in_pixeldir-pngtEXtchecksum\x00checksum-png') | 266 expected_image='image_not_in_pixeldir-pngtEXtchecksum\x00checksum-
png') |
| 261 | 267 |
| 262 # For testing that virtual test suites don't expand names containing themsel
ves | 268 # For testing that virtual test suites don't expand names containing themsel
ves |
| 263 # See webkit.org/b/97925 and base_unittest.PortTest.test_tests(). | 269 # See webkit.org/b/97925 and base_unittest.PortTest.test_tests(). |
| 264 tests.add('passes/test-virtual-passes.html') | 270 tests.add('passes/test-virtual-passes.html') |
| 265 tests.add('passes/virtual_passes/test-virtual-passes.html') | 271 tests.add('passes/virtual_passes/test-virtual-passes.html') |
| 266 | 272 |
| 267 return tests | 273 return tests |
| 268 | 274 |
| 269 | |
| 270 # Here we use a non-standard location for the layout tests, to ensure that | 275 # Here we use a non-standard location for the layout tests, to ensure that |
| 271 # this works. The path contains a '.' in the name because we've seen bugs | 276 # this works. The path contains a '.' in the name because we've seen bugs |
| 272 # related to this before. | 277 # related to this before. |
| 273 | 278 |
| 274 LAYOUT_TEST_DIR = '/test.checkout/LayoutTests' | 279 LAYOUT_TEST_DIR = '/test.checkout/LayoutTests' |
| 275 PERF_TEST_DIR = '/test.checkout/PerformanceTests' | 280 PERF_TEST_DIR = '/test.checkout/PerformanceTests' |
| 276 | 281 |
| 277 | 282 |
| 278 # Here we synthesize an in-memory filesystem from the test list | 283 # Here we synthesize an in-memory filesystem from the test list |
| 279 # in order to fully control the test output and to demonstrate that | 284 # in order to fully control the test output and to demonstrate that |
| (...skipping 66 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 346 for test in test_list.tests.values(): | 351 for test in test_list.tests.values(): |
| 347 add_file(test, test.name[test.name.rfind('.'):], '') | 352 add_file(test, test.name[test.name.rfind('.'):], '') |
| 348 if test.is_reftest: | 353 if test.is_reftest: |
| 349 continue | 354 continue |
| 350 if test.actual_audio: | 355 if test.actual_audio: |
| 351 add_file(test, '-expected.wav', test.expected_audio) | 356 add_file(test, '-expected.wav', test.expected_audio) |
| 352 continue | 357 continue |
| 353 add_file(test, '-expected.txt', test.expected_text) | 358 add_file(test, '-expected.txt', test.expected_text) |
| 354 add_file(test, '-expected.png', test.expected_image) | 359 add_file(test, '-expected.png', test.expected_image) |
| 355 | 360 |
| 356 filesystem.write_text_file(filesystem.join(LAYOUT_TEST_DIR, 'virtual', 'virt
ual_passes', 'passes', 'args-expected.txt'), 'args-txt --virtual-arg') | 361 filesystem.write_text_file( |
| 362 filesystem.join(LAYOUT_TEST_DIR, 'virtual', 'virtual_passes', 'passes',
'args-expected.txt'), 'args-txt --virtual-arg') |
| 357 # Clear the list of written files so that we can watch what happens during t
esting. | 363 # Clear the list of written files so that we can watch what happens during t
esting. |
| 358 filesystem.clear_written_files() | 364 filesystem.clear_written_files() |
| 359 | 365 |
| 360 | 366 |
| 361 class TestPort(Port): | 367 class TestPort(Port): |
| 362 port_name = 'test' | 368 port_name = 'test' |
| 363 default_port_name = 'test-mac-mac10.10' | 369 default_port_name = 'test-mac-mac10.10' |
| 364 | 370 |
| 365 # TODO(wkorman): The below constant is legacy code and is only referenced by
a unit test. Find the modern way to do | 371 # TODO(wkorman): The below constant is legacy code and is only referenced by
a unit test. Find the modern way to do |
| 366 # the same thing that test is doing and delete this. | 372 # the same thing that test is doing and delete this. |
| 367 # | 373 # |
| 368 # A list of platform names sufficient to cover all the baselines. | 374 # A list of platform names sufficient to cover all the baselines. |
| 369 # The list should be sorted so that a later platform will reuse | 375 # The list should be sorted so that a later platform will reuse |
| 370 # an earlier platform's baselines if they are the same (e.g., | 376 # an earlier platform's baselines if they are the same (e.g., |
| 371 # 'mac10.10' should precede 'mac10.9'). | 377 # 'mac10.10' should precede 'mac10.9'). |
| 372 ALL_BASELINE_VARIANTS = ( | 378 ALL_BASELINE_VARIANTS = ('test-linux-trusty', 'test-linux-precise', 'test-li
nux-x86', 'test-mac-mac10.11', 'test-mac-mac10.10', |
| 373 'test-linux-trusty', 'test-linux-precise', 'test-linux-x86', | 379 'test-win-win10', 'test-win-win7') |
| 374 'test-mac-mac10.11', 'test-mac-mac10.10', | |
| 375 'test-win-win10', 'test-win-win7' | |
| 376 ) | |
| 377 | 380 |
| 378 FALLBACK_PATHS = { | 381 FALLBACK_PATHS = { |
| 379 'win7': ['test-win-win7', 'test-win-win10'], | 382 'win7': ['test-win-win7', 'test-win-win10'], |
| 380 'win10': ['test-win-win10'], | 383 'win10': ['test-win-win10'], |
| 381 'mac10.10': ['test-mac-mac10.10', 'test-mac-mac10.11'], | 384 'mac10.10': ['test-mac-mac10.10', 'test-mac-mac10.11'], |
| 382 'mac10.11': ['test-mac-mac10.11'], | 385 'mac10.11': ['test-mac-mac10.11'], |
| 383 'trusty': ['test-linux-trusty', 'test-win-win7'], | 386 'trusty': ['test-linux-trusty', 'test-win-win7'], |
| 384 'precise': ['test-linux-precise', 'test-linux-trusty', 'test-win-win
7'], | 387 'precise': ['test-linux-precise', 'test-linux-trusty', 'test-win-win7'], |
| 385 'linux32': ['test-linux-x86', 'test-linux-precise', 'test-linux-trus
ty', 'test-win-win7'], | 388 'linux32': ['test-linux-x86', 'test-linux-precise', 'test-linux-trusty',
'test-win-win7'], |
| 386 } | 389 } |
| 387 | 390 |
| 388 @classmethod | 391 @classmethod |
| 389 def determine_full_port_name(cls, host, options, port_name): | 392 def determine_full_port_name(cls, host, options, port_name): |
| 390 if port_name == 'test': | 393 if port_name == 'test': |
| 391 return TestPort.default_port_name | 394 return TestPort.default_port_name |
| 392 return port_name | 395 return port_name |
| 393 | 396 |
| 394 def __init__(self, host, port_name=None, **kwargs): | 397 def __init__(self, host, port_name=None, **kwargs): |
| 395 Port.__init__(self, host, port_name or TestPort.default_port_name, **kwa
rgs) | 398 Port.__init__(self, host, port_name or TestPort.default_port_name, **kwa
rgs) |
| (...skipping 68 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 464 def layout_tests_dir(self): | 467 def layout_tests_dir(self): |
| 465 return LAYOUT_TEST_DIR | 468 return LAYOUT_TEST_DIR |
| 466 | 469 |
| 467 def perf_tests_dir(self): | 470 def perf_tests_dir(self): |
| 468 return PERF_TEST_DIR | 471 return PERF_TEST_DIR |
| 469 | 472 |
| 470 def webkit_base(self): | 473 def webkit_base(self): |
| 471 return '/test.checkout' | 474 return '/test.checkout' |
| 472 | 475 |
| 473 def _skipped_tests_for_unsupported_features(self, test_list): | 476 def _skipped_tests_for_unsupported_features(self, test_list): |
| 474 return set(['failures/expected/skip_text.html', | 477 return set(['failures/expected/skip_text.html', 'failures/unexpected/ski
p_pass.html', 'virtual/skipped/failures/expected']) |
| 475 'failures/unexpected/skip_pass.html', | |
| 476 'virtual/skipped/failures/expected']) | |
| 477 | 478 |
| 478 def name(self): | 479 def name(self): |
| 479 return self._name | 480 return self._name |
| 480 | 481 |
| 481 def operating_system(self): | 482 def operating_system(self): |
| 482 return self._operating_system | 483 return self._operating_system |
| 483 | 484 |
| 484 def _path_to_wdiff(self): | 485 def _path_to_wdiff(self): |
| 485 return None | 486 return None |
| 486 | 487 |
| (...skipping 36 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 523 def _port_specific_expectations_files(self): | 524 def _port_specific_expectations_files(self): |
| 524 return [self._filesystem.join(self._webkit_baseline_path(d), 'TestExpect
ations') for d in ['test', 'test-win-win7']] | 525 return [self._filesystem.join(self._webkit_baseline_path(d), 'TestExpect
ations') for d in ['test', 'test-win-win7']] |
| 525 | 526 |
| 526 def all_test_configurations(self): | 527 def all_test_configurations(self): |
| 527 """Returns a sequence of the TestConfigurations the port supports.""" | 528 """Returns a sequence of the TestConfigurations the port supports.""" |
| 528 # By default, we assume we want to test every graphics type in | 529 # By default, we assume we want to test every graphics type in |
| 529 # every configuration on every system. | 530 # every configuration on every system. |
| 530 test_configurations = [] | 531 test_configurations = [] |
| 531 for version, architecture in self._all_systems(): | 532 for version, architecture in self._all_systems(): |
| 532 for build_type in self._all_build_types(): | 533 for build_type in self._all_build_types(): |
| 533 test_configurations.append(TestConfiguration( | 534 test_configurations.append(TestConfiguration(version=version, ar
chitecture=architecture, build_type=build_type)) |
| 534 version=version, | |
| 535 architecture=architecture, | |
| 536 build_type=build_type)) | |
| 537 return test_configurations | 535 return test_configurations |
| 538 | 536 |
| 539 def _all_systems(self): | 537 def _all_systems(self): |
| 540 return (('mac10.10', 'x86'), | 538 return (('mac10.10', 'x86'), ('mac10.11', 'x86'), ('win7', 'x86'), ('win
10', 'x86'), ('linux32', 'x86'), |
| 541 ('mac10.11', 'x86'), | 539 ('precise', 'x86_64'), ('trusty', 'x86_64')) |
| 542 ('win7', 'x86'), | |
| 543 ('win10', 'x86'), | |
| 544 ('linux32', 'x86'), | |
| 545 ('precise', 'x86_64'), | |
| 546 ('trusty', 'x86_64')) | |
| 547 | 540 |
| 548 def _all_build_types(self): | 541 def _all_build_types(self): |
| 549 return ('debug', 'release') | 542 return ('debug', 'release') |
| 550 | 543 |
| 551 def configuration_specifier_macros(self): | 544 def configuration_specifier_macros(self): |
| 552 """To avoid surprises when introducing new macros, these are intentional
ly fixed in time.""" | 545 """To avoid surprises when introducing new macros, these are intentional
ly fixed in time.""" |
| 553 return { | 546 return {'mac': ['mac10.10', 'mac10.11'], 'win': ['win7', 'win10'], 'linu
x': ['linux32', 'precise', 'trusty']} |
| 554 'mac': ['mac10.10', 'mac10.11'], | |
| 555 'win': ['win7', 'win10'], | |
| 556 'linux': ['linux32', 'precise', 'trusty'] | |
| 557 } | |
| 558 | 547 |
| 559 def virtual_test_suites(self): | 548 def virtual_test_suites(self): |
| 560 return [ | 549 return [ |
| 561 VirtualTestSuite(prefix='virtual_passes', base='passes', args=['--vi
rtual-arg']), | 550 VirtualTestSuite(prefix='virtual_passes', |
| 562 VirtualTestSuite(prefix='skipped', base='failures/expected', args=['
--virtual-arg2']), | 551 base='passes', |
| 563 VirtualTestSuite(prefix='references_use_default_args', base='passes/
reftest.html', | 552 args=['--virtual-arg']), |
| 564 args=['--virtual-arg'], references_use_default_args
=True), | 553 VirtualTestSuite(prefix='skipped', |
| 554 base='failures/expected', |
| 555 args=['--virtual-arg2']), |
| 556 VirtualTestSuite(prefix='references_use_default_args', |
| 557 base='passes/reftest.html', |
| 558 args=['--virtual-arg'], |
| 559 references_use_default_args=True), |
| 565 ] | 560 ] |
| 566 | 561 |
| 567 | 562 |
| 568 class TestDriver(Driver): | 563 class TestDriver(Driver): |
| 569 """Test/Dummy implementation of the driver interface.""" | 564 """Test/Dummy implementation of the driver interface.""" |
| 570 next_pid = 1 | 565 next_pid = 1 |
| 571 | 566 |
| 572 def __init__(self, *args, **kwargs): | 567 def __init__(self, *args, **kwargs): |
| 573 super(TestDriver, self).__init__(*args, **kwargs) | 568 super(TestDriver, self).__init__(*args, **kwargs) |
| 574 self.started = False | 569 self.started = False |
| 575 self.pid = 0 | 570 self.pid = 0 |
| 576 | 571 |
| 577 def cmd_line(self, pixel_tests, per_test_args): | 572 def cmd_line(self, pixel_tests, per_test_args): |
| 578 pixel_tests_flag = '-p' if pixel_tests else '' | 573 pixel_tests_flag = '-p' if pixel_tests else '' |
| 579 return [self._port._path_to_driver()] + [pixel_tests_flag] + self._port.
get_option('additional_driver_flag', []) + per_test_args | 574 return [self._port._path_to_driver()] + [pixel_tests_flag] + self._port.
get_option('additional_driver_flag', |
| 575
[]) + per_test_args |
| 580 | 576 |
| 581 def run_test(self, driver_input, stop_when_done): | 577 def run_test(self, driver_input, stop_when_done): |
| 582 if not self.started: | 578 if not self.started: |
| 583 self.started = True | 579 self.started = True |
| 584 self.pid = TestDriver.next_pid | 580 self.pid = TestDriver.next_pid |
| 585 TestDriver.next_pid += 1 | 581 TestDriver.next_pid += 1 |
| 586 | 582 |
| 587 start_time = time.time() | 583 start_time = time.time() |
| 588 test_name = driver_input.test_name | 584 test_name = driver_input.test_name |
| 589 test_args = driver_input.args or [] | 585 test_args = driver_input.args or [] |
| (...skipping 57 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 647 crash = True | 643 crash = True |
| 648 crash_log = 'reftest crash log' | 644 crash_log = 'reftest crash log' |
| 649 | 645 |
| 650 if stop_when_done: | 646 if stop_when_done: |
| 651 self.stop() | 647 self.stop() |
| 652 | 648 |
| 653 if test.actual_checksum == driver_input.image_hash: | 649 if test.actual_checksum == driver_input.image_hash: |
| 654 image = None | 650 image = None |
| 655 else: | 651 else: |
| 656 image = test.actual_image | 652 image = test.actual_image |
| 657 return DriverOutput(actual_text, image, test.actual_checksum, audio, | 653 return DriverOutput(actual_text, |
| 658 crash=(crash or web_process_crash), crashed_process_name=crashed_pro
cess_name, | 654 image, |
| 659 crashed_pid=crashed_pid, crash_log=crash_log, | 655 test.actual_checksum, |
| 660 test_time=time.time() - start_time, timeout=test.timeout, error=test
.error, pid=self.pid, | 656 audio, |
| 661 leak=test.leak) | 657 crash=(crash or web_process_crash), |
| 658 crashed_process_name=crashed_process_name, |
| 659 crashed_pid=crashed_pid, |
| 660 crash_log=crash_log, |
| 661 test_time=time.time() - start_time, |
| 662 timeout=test.timeout, |
| 663 error=test.error, |
| 664 pid=self.pid, |
| 665 leak=test.leak) |
| 662 | 666 |
| 663 def stop(self): | 667 def stop(self): |
| 664 self.started = False | 668 self.started = False |
| OLD | NEW |