OLD | NEW |
1 # Copyright (C) 2010 Google Inc. All rights reserved. | 1 # Copyright (C) 2010 Google Inc. All rights reserved. |
2 # | 2 # |
3 # Redistribution and use in source and binary forms, with or without | 3 # Redistribution and use in source and binary forms, with or without |
4 # modification, are permitted provided that the following conditions are | 4 # modification, are permitted provided that the following conditions are |
5 # met: | 5 # met: |
6 # | 6 # |
7 # * Redistributions of source code must retain the above copyright | 7 # * Redistributions of source code must retain the above copyright |
8 # notice, this list of conditions and the following disclaimer. | 8 # notice, this list of conditions and the following disclaimer. |
9 # * Redistributions in binary form must reproduce the above | 9 # * Redistributions in binary form must reproduce the above |
10 # copyright notice, this list of conditions and the following disclaimer | 10 # copyright notice, this list of conditions and the following disclaimer |
(...skipping 24 matching lines...) Expand all Loading... |
35 from webkitpy.layout_tests.port.driver import DeviceFailure, Driver, DriverOutpu
t | 35 from webkitpy.layout_tests.port.driver import DeviceFailure, Driver, DriverOutpu
t |
36 from webkitpy.layout_tests.models.test_configuration import TestConfiguration | 36 from webkitpy.layout_tests.models.test_configuration import TestConfiguration |
37 from webkitpy.layout_tests.models import test_run_results | 37 from webkitpy.layout_tests.models import test_run_results |
38 from webkitpy.common.system.filesystem_mock import MockFileSystem | 38 from webkitpy.common.system.filesystem_mock import MockFileSystem |
39 from webkitpy.common.system.crashlogs import CrashLogs | 39 from webkitpy.common.system.crashlogs import CrashLogs |
40 | 40 |
41 | 41 |
42 # This sets basic expectations for a test. Each individual expectation | 42 # This sets basic expectations for a test. Each individual expectation |
43 # can be overridden by a keyword argument in TestList.add(). | 43 # can be overridden by a keyword argument in TestList.add(). |
44 class TestInstance(object): | 44 class TestInstance(object): |
| 45 |
45 def __init__(self, name): | 46 def __init__(self, name): |
46 self.name = name | 47 self.name = name |
47 self.base = name[(name.rfind("/") + 1):name.rfind(".")] | 48 self.base = name[(name.rfind("/") + 1):name.rfind(".")] |
48 self.crash = False | 49 self.crash = False |
49 self.web_process_crash = False | 50 self.web_process_crash = False |
50 self.exception = False | 51 self.exception = False |
51 self.keyboard = False | 52 self.keyboard = False |
52 self.error = '' | 53 self.error = '' |
53 self.timeout = False | 54 self.timeout = False |
54 self.is_reftest = False | 55 self.is_reftest = False |
(...skipping 13 matching lines...) Expand all Loading... |
68 self.expected_text = self.actual_text | 69 self.expected_text = self.actual_text |
69 self.expected_image = self.actual_image | 70 self.expected_image = self.actual_image |
70 | 71 |
71 self.actual_audio = None | 72 self.actual_audio = None |
72 self.expected_audio = None | 73 self.expected_audio = None |
73 | 74 |
74 | 75 |
75 # This is an in-memory list of tests, what we want them to produce, and | 76 # This is an in-memory list of tests, what we want them to produce, and |
76 # what we want to claim are the expected results. | 77 # what we want to claim are the expected results. |
77 class TestList(object): | 78 class TestList(object): |
| 79 |
78 def __init__(self): | 80 def __init__(self): |
79 self.tests = {} | 81 self.tests = {} |
80 | 82 |
81 def add(self, name, **kwargs): | 83 def add(self, name, **kwargs): |
82 test = TestInstance(name) | 84 test = TestInstance(name) |
83 for key, value in kwargs.items(): | 85 for key, value in kwargs.items(): |
84 test.__dict__[key] = value | 86 test.__dict__[key] = value |
85 self.tests[name] = test | 87 self.tests[name] = test |
86 | 88 |
87 def add_reftest(self, name, reference_name, same_image, crash=False): | 89 def add_reftest(self, name, reference_name, same_image, crash=False): |
(...skipping 14 matching lines...) Expand all Loading... |
102 | 104 |
103 # | 105 # |
104 # These numbers may need to be updated whenever we add or delete tests. This inc
ludes virtual tests. | 106 # These numbers may need to be updated whenever we add or delete tests. This inc
ludes virtual tests. |
105 # | 107 # |
106 TOTAL_TESTS = 114 | 108 TOTAL_TESTS = 114 |
107 TOTAL_SKIPS = 26 | 109 TOTAL_SKIPS = 26 |
108 TOTAL_CRASHES = 80 | 110 TOTAL_CRASHES = 80 |
109 UNEXPECTED_PASSES = 1 | 111 UNEXPECTED_PASSES = 1 |
110 UNEXPECTED_FAILURES = 26 | 112 UNEXPECTED_FAILURES = 26 |
111 | 113 |
| 114 |
112 def unit_test_list(): | 115 def unit_test_list(): |
113 tests = TestList() | 116 tests = TestList() |
114 tests.add('failures/expected/crash.html', crash=True) | 117 tests.add('failures/expected/crash.html', crash=True) |
115 tests.add('failures/expected/exception.html', exception=True) | 118 tests.add('failures/expected/exception.html', exception=True) |
116 tests.add('failures/expected/device_failure.html', device_failure=True) | 119 tests.add('failures/expected/device_failure.html', device_failure=True) |
117 tests.add('failures/expected/timeout.html', timeout=True) | 120 tests.add('failures/expected/timeout.html', timeout=True) |
118 tests.add('failures/expected/leak.html', leak=True) | 121 tests.add('failures/expected/leak.html', leak=True) |
119 tests.add('failures/expected/missing_text.html', expected_text=None) | 122 tests.add('failures/expected/missing_text.html', expected_text=None) |
120 tests.add('failures/expected/needsrebaseline.html', actual_text='needsrebase
line text') | 123 tests.add('failures/expected/needsrebaseline.html', actual_text='needsrebase
line text') |
121 tests.add('failures/expected/needsmanualrebaseline.html', actual_text='needs
manualrebaseline text') | 124 tests.add('failures/expected/needsmanualrebaseline.html', actual_text='needs
manualrebaseline text') |
(...skipping 88 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
210 | 213 |
211 # This adds a different virtual reference to ensure that that also works. | 214 # This adds a different virtual reference to ensure that that also works. |
212 tests.add('virtual/virtual_passes/passes/reftest-expected.html', actual_chec
ksum='xxx', actual_image='XXX', is_reftest=True) | 215 tests.add('virtual/virtual_passes/passes/reftest-expected.html', actual_chec
ksum='xxx', actual_image='XXX', is_reftest=True) |
213 | 216 |
214 tests.add_reftest('passes/mismatch.html', 'passes/mismatch-expected-mismatch
.html', same_image=False) | 217 tests.add_reftest('passes/mismatch.html', 'passes/mismatch-expected-mismatch
.html', same_image=False) |
215 tests.add_reftest('passes/svgreftest.svg', 'passes/svgreftest-expected.svg',
same_image=True) | 218 tests.add_reftest('passes/svgreftest.svg', 'passes/svgreftest-expected.svg',
same_image=True) |
216 tests.add_reftest('passes/xhtreftest.xht', 'passes/xhtreftest-expected.html'
, same_image=True) | 219 tests.add_reftest('passes/xhtreftest.xht', 'passes/xhtreftest-expected.html'
, same_image=True) |
217 tests.add_reftest('passes/phpreftest.php', 'passes/phpreftest-expected-misma
tch.svg', same_image=False) | 220 tests.add_reftest('passes/phpreftest.php', 'passes/phpreftest-expected-misma
tch.svg', same_image=False) |
218 tests.add_reftest('failures/expected/reftest.html', 'failures/expected/refte
st-expected.html', same_image=False) | 221 tests.add_reftest('failures/expected/reftest.html', 'failures/expected/refte
st-expected.html', same_image=False) |
219 tests.add_reftest('failures/expected/mismatch.html', 'failures/expected/mism
atch-expected-mismatch.html', same_image=True) | 222 tests.add_reftest('failures/expected/mismatch.html', 'failures/expected/mism
atch-expected-mismatch.html', same_image=True) |
220 tests.add_reftest('failures/unexpected/crash-reftest.html', 'failures/unexpe
cted/crash-reftest-expected.html', same_image=True, crash=True) | 223 tests.add_reftest('failures/unexpected/crash-reftest.html', |
| 224 'failures/unexpected/crash-reftest-expected.html', same_im
age=True, crash=True) |
221 tests.add_reftest('failures/unexpected/reftest.html', 'failures/unexpected/r
eftest-expected.html', same_image=False) | 225 tests.add_reftest('failures/unexpected/reftest.html', 'failures/unexpected/r
eftest-expected.html', same_image=False) |
222 tests.add_reftest('failures/unexpected/mismatch.html', 'failures/unexpected/
mismatch-expected-mismatch.html', same_image=True) | 226 tests.add_reftest('failures/unexpected/mismatch.html', 'failures/unexpected/
mismatch-expected-mismatch.html', same_image=True) |
223 tests.add('failures/unexpected/reftest-nopixel.html', actual_checksum=None,
actual_image=None, is_reftest=True) | 227 tests.add('failures/unexpected/reftest-nopixel.html', actual_checksum=None,
actual_image=None, is_reftest=True) |
224 tests.add('failures/unexpected/reftest-nopixel-expected.html', actual_checks
um=None, actual_image=None, is_reftest=True) | 228 tests.add('failures/unexpected/reftest-nopixel-expected.html', actual_checks
um=None, actual_image=None, is_reftest=True) |
225 tests.add('reftests/foo/test.html') | 229 tests.add('reftests/foo/test.html') |
226 tests.add('reftests/foo/test-ref.html') | 230 tests.add('reftests/foo/test-ref.html') |
227 | 231 |
228 tests.add('reftests/foo/multiple-match-success.html', actual_checksum='abc',
actual_image='abc') | 232 tests.add('reftests/foo/multiple-match-success.html', actual_checksum='abc',
actual_image='abc') |
229 tests.add('reftests/foo/multiple-match-failure.html', actual_checksum='abc',
actual_image='abc') | 233 tests.add('reftests/foo/multiple-match-failure.html', actual_checksum='abc',
actual_image='abc') |
230 tests.add('reftests/foo/multiple-mismatch-success.html', actual_checksum='ab
c', actual_image='abc') | 234 tests.add('reftests/foo/multiple-mismatch-success.html', actual_checksum='ab
c', actual_image='abc') |
(...skipping 15 matching lines...) Expand all Loading... |
246 # For testing that we don't run tests under platform/. Note that these don't
contribute to TOTAL_TESTS. | 250 # For testing that we don't run tests under platform/. Note that these don't
contribute to TOTAL_TESTS. |
247 tests.add('platform/test-mac-10.10/http/test.html') | 251 tests.add('platform/test-mac-10.10/http/test.html') |
248 tests.add('platform/test-win-win7/http/test.html') | 252 tests.add('platform/test-win-win7/http/test.html') |
249 | 253 |
250 # For testing if perf tests are running in a locked shard. | 254 # For testing if perf tests are running in a locked shard. |
251 tests.add('perf/foo/test.html') | 255 tests.add('perf/foo/test.html') |
252 tests.add('perf/foo/test-ref.html') | 256 tests.add('perf/foo/test-ref.html') |
253 | 257 |
254 # For testing --pixel-test-directories. | 258 # For testing --pixel-test-directories. |
255 tests.add('failures/unexpected/pixeldir/image_in_pixeldir.html', | 259 tests.add('failures/unexpected/pixeldir/image_in_pixeldir.html', |
256 actual_image='image_in_pixeldir-pngtEXtchecksum\x00checksum_fail', | 260 actual_image='image_in_pixeldir-pngtEXtchecksum\x00checksum_fail', |
257 expected_image='image_in_pixeldir-pngtEXtchecksum\x00checksum-png') | 261 expected_image='image_in_pixeldir-pngtEXtchecksum\x00checksum-png'
) |
258 tests.add('failures/unexpected/image_not_in_pixeldir.html', | 262 tests.add('failures/unexpected/image_not_in_pixeldir.html', |
259 actual_image='image_not_in_pixeldir-pngtEXtchecksum\x00checksum_fail', | 263 actual_image='image_not_in_pixeldir-pngtEXtchecksum\x00checksum_fa
il', |
260 expected_image='image_not_in_pixeldir-pngtEXtchecksum\x00checksum-png') | 264 expected_image='image_not_in_pixeldir-pngtEXtchecksum\x00checksum-
png') |
261 | 265 |
262 # For testing that virtual test suites don't expand names containing themsel
ves | 266 # For testing that virtual test suites don't expand names containing themsel
ves |
263 # See webkit.org/b/97925 and base_unittest.PortTest.test_tests(). | 267 # See webkit.org/b/97925 and base_unittest.PortTest.test_tests(). |
264 tests.add('passes/test-virtual-passes.html') | 268 tests.add('passes/test-virtual-passes.html') |
265 tests.add('passes/virtual_passes/test-virtual-passes.html') | 269 tests.add('passes/virtual_passes/test-virtual-passes.html') |
266 | 270 |
267 return tests | 271 return tests |
268 | 272 |
269 | 273 |
270 # Here we use a non-standard location for the layout tests, to ensure that | 274 # Here we use a non-standard location for the layout tests, to ensure that |
(...skipping 75 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
346 for test in test_list.tests.values(): | 350 for test in test_list.tests.values(): |
347 add_file(test, test.name[test.name.rfind('.'):], '') | 351 add_file(test, test.name[test.name.rfind('.'):], '') |
348 if test.is_reftest: | 352 if test.is_reftest: |
349 continue | 353 continue |
350 if test.actual_audio: | 354 if test.actual_audio: |
351 add_file(test, '-expected.wav', test.expected_audio) | 355 add_file(test, '-expected.wav', test.expected_audio) |
352 continue | 356 continue |
353 add_file(test, '-expected.txt', test.expected_text) | 357 add_file(test, '-expected.txt', test.expected_text) |
354 add_file(test, '-expected.png', test.expected_image) | 358 add_file(test, '-expected.png', test.expected_image) |
355 | 359 |
356 filesystem.write_text_file(filesystem.join(LAYOUT_TEST_DIR, 'virtual', 'virt
ual_passes', 'passes', 'args-expected.txt'), 'args-txt --virtual-arg') | 360 filesystem.write_text_file(filesystem.join(LAYOUT_TEST_DIR, 'virtual', 'virt
ual_passes', |
| 361 'passes', 'args-expected.txt'), '
args-txt --virtual-arg') |
357 # Clear the list of written files so that we can watch what happens during t
esting. | 362 # Clear the list of written files so that we can watch what happens during t
esting. |
358 filesystem.clear_written_files() | 363 filesystem.clear_written_files() |
359 | 364 |
360 | 365 |
361 class TestPort(Port): | 366 class TestPort(Port): |
362 port_name = 'test' | 367 port_name = 'test' |
363 default_port_name = 'test-mac-mac10.10' | 368 default_port_name = 'test-mac-mac10.10' |
364 | 369 |
365 # TODO(wkorman): The below constant is legacy code and is only referenced by
a unit test. Find the modern way to do | 370 # TODO(wkorman): The below constant is legacy code and is only referenced by
a unit test. Find the modern way to do |
366 # the same thing that test is doing and delete this. | 371 # the same thing that test is doing and delete this. |
367 # | 372 # |
368 # A list of platform names sufficient to cover all the baselines. | 373 # A list of platform names sufficient to cover all the baselines. |
369 # The list should be sorted so that a later platform will reuse | 374 # The list should be sorted so that a later platform will reuse |
370 # an earlier platform's baselines if they are the same (e.g., | 375 # an earlier platform's baselines if they are the same (e.g., |
371 # 'mac10.10' should precede 'mac10.9'). | 376 # 'mac10.10' should precede 'mac10.9'). |
372 ALL_BASELINE_VARIANTS = ( | 377 ALL_BASELINE_VARIANTS = ( |
373 'test-linux-trusty', 'test-linux-precise', 'test-linux-x86', | 378 'test-linux-trusty', 'test-linux-precise', 'test-linux-x86', |
374 'test-mac-mac10.11', 'test-mac-mac10.10', | 379 'test-mac-mac10.11', 'test-mac-mac10.10', |
375 'test-win-win10', 'test-win-win7' | 380 'test-win-win10', 'test-win-win7' |
376 ) | 381 ) |
377 | 382 |
378 FALLBACK_PATHS = { | 383 FALLBACK_PATHS = { |
379 'win7': ['test-win-win7', 'test-win-win10'], | 384 'win7': ['test-win-win7', 'test-win-win10'], |
380 'win10': ['test-win-win10'], | 385 'win10': ['test-win-win10'], |
381 'mac10.10': ['test-mac-mac10.10', 'test-mac-mac10.11'], | 386 'mac10.10': ['test-mac-mac10.10', 'test-mac-mac10.11'], |
382 'mac10.11': ['test-mac-mac10.11'], | 387 'mac10.11': ['test-mac-mac10.11'], |
383 'trusty': ['test-linux-trusty', 'test-win-win7'], | 388 'trusty': ['test-linux-trusty', 'test-win-win7'], |
384 'precise': ['test-linux-precise', 'test-linux-trusty', 'test-win-win
7'], | 389 'precise': ['test-linux-precise', 'test-linux-trusty', 'test-win-win7'], |
385 'linux32': ['test-linux-x86', 'test-linux-precise', 'test-linux-trus
ty', 'test-win-win7'], | 390 'linux32': ['test-linux-x86', 'test-linux-precise', 'test-linux-trusty',
'test-win-win7'], |
386 } | 391 } |
387 | 392 |
388 @classmethod | 393 @classmethod |
389 def determine_full_port_name(cls, host, options, port_name): | 394 def determine_full_port_name(cls, host, options, port_name): |
390 if port_name == 'test': | 395 if port_name == 'test': |
391 return TestPort.default_port_name | 396 return TestPort.default_port_name |
392 return port_name | 397 return port_name |
393 | 398 |
394 def __init__(self, host, port_name=None, **kwargs): | 399 def __init__(self, host, port_name=None, **kwargs): |
395 Port.__init__(self, host, port_name or TestPort.default_port_name, **kwa
rgs) | 400 Port.__init__(self, host, port_name or TestPort.default_port_name, **kwa
rgs) |
(...skipping 252 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
648 crash_log = 'reftest crash log' | 653 crash_log = 'reftest crash log' |
649 | 654 |
650 if stop_when_done: | 655 if stop_when_done: |
651 self.stop() | 656 self.stop() |
652 | 657 |
653 if test.actual_checksum == driver_input.image_hash: | 658 if test.actual_checksum == driver_input.image_hash: |
654 image = None | 659 image = None |
655 else: | 660 else: |
656 image = test.actual_image | 661 image = test.actual_image |
657 return DriverOutput(actual_text, image, test.actual_checksum, audio, | 662 return DriverOutput(actual_text, image, test.actual_checksum, audio, |
658 crash=(crash or web_process_crash), crashed_process_name=crashed_pro
cess_name, | 663 crash=(crash or web_process_crash), crashed_process_
name=crashed_process_name, |
659 crashed_pid=crashed_pid, crash_log=crash_log, | 664 crashed_pid=crashed_pid, crash_log=crash_log, |
660 test_time=time.time() - start_time, timeout=test.timeout, error=test
.error, pid=self.pid, | 665 test_time=time.time() - start_time, timeout=test.tim
eout, error=test.error, pid=self.pid, |
661 leak=test.leak) | 666 leak=test.leak) |
662 | 667 |
663 def stop(self): | 668 def stop(self): |
664 self.started = False | 669 self.started = False |
OLD | NEW |