| OLD | NEW |
| 1 # Copyright 2016 The Chromium Authors. All rights reserved. | 1 # Copyright 2016 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 import optparse | 5 import optparse |
| 6 import unittest | 6 import unittest |
| 7 | 7 |
| 8 from webkitpy.common.net.buildbot import Build | 8 from webkitpy.common.net.buildbot import Build |
| 9 from webkitpy.common.net.layouttestresults import LayoutTestResults | 9 from webkitpy.common.net.layouttestresults import LayoutTestResults |
| 10 from webkitpy.common.system.executive_mock import MockExecutive, MockExecutive2 | 10 from webkitpy.common.system.executive_mock import MockExecutive, MockExecutive2 |
| 11 from webkitpy.common.system.outputcapture import OutputCapture | 11 from webkitpy.common.system.outputcapture import OutputCapture |
| 12 from webkitpy.layout_tests.builder_list import BuilderList | 12 from webkitpy.layout_tests.builder_list import BuilderList |
| 13 from webkitpy.tool.commands.rebaseline import ( | 13 from webkitpy.tool.commands.rebaseline import ( |
| 14 AbstractParallelRebaselineCommand, CopyExistingBaselinesInternal, | 14 AbstractParallelRebaselineCommand, CopyExistingBaselinesInternal, |
| 15 Rebaseline, RebaselineExpectations, RebaselineJson, RebaselineTest, ChangeSe
t | 15 Rebaseline, RebaselineExpectations, RebaselineJson, RebaselineTest |
| 16 ) | 16 ) |
| 17 from webkitpy.tool.mock_tool import MockWebKitPatch | 17 from webkitpy.tool.mock_tool import MockWebKitPatch |
| 18 | 18 |
| 19 | 19 |
| 20 # pylint: disable=protected-access | 20 # pylint: disable=protected-access |
| 21 class BaseTestCase(unittest.TestCase): | 21 class BaseTestCase(unittest.TestCase): |
| 22 MOCK_WEB_RESULT = 'MOCK Web result, convert 404 to None=True' | 22 MOCK_WEB_RESULT = 'MOCK Web result, convert 404 to None=True' |
| 23 WEB_PREFIX = 'https://storage.googleapis.com/chromium-layout-test-archives/M
OCK_Mac10_11/results/layout-test-results' | 23 WEB_PREFIX = 'https://storage.googleapis.com/chromium-layout-test-archives/M
OCK_Mac10_11/results/layout-test-results' |
| 24 | 24 |
| 25 command_constructor = None | 25 command_constructor = None |
| (...skipping 98 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 124 options = optparse.Values({ | 124 options = optparse.Values({ |
| 125 'builder': "MOCK Mac10.11", | 125 'builder': "MOCK Mac10.11", |
| 126 'suffixes': 'txt', | 126 'suffixes': 'txt', |
| 127 'verbose': True, | 127 'verbose': True, |
| 128 'test': "failures/expected/image.html", | 128 'test': "failures/expected/image.html", |
| 129 'results_directory': None | 129 'results_directory': None |
| 130 }) | 130 }) |
| 131 oc.capture_output() | 131 oc.capture_output() |
| 132 self.command.execute(options, [], self.tool) | 132 self.command.execute(options, [], self.tool) |
| 133 finally: | 133 finally: |
| 134 out, _, _ = oc.restore_output() | 134 oc.restore_output() |
| 135 | 135 |
| 136 self.assertMultiLineEqual( | 136 self.assertMultiLineEqual( |
| 137 self._read(self.tool.filesystem.join( | 137 self._read(self.tool.filesystem.join( |
| 138 port.layout_tests_dir(), | 138 port.layout_tests_dir(), |
| 139 'platform/test-mac-mac10.10/failures/expected/image-expected.txt
')), | 139 'platform/test-mac-mac10.10/failures/expected/image-expected.txt
')), |
| 140 'original mac10.11 result') | 140 'original mac10.11 result') |
| 141 self.assertMultiLineEqual(out, '{"add": [], "remove-lines": [], "delete"
: []}\n') | |
| 142 | 141 |
| 143 def test_copying_overwritten_baseline_to_multiple_locations(self): | 142 def test_copying_overwritten_baseline_to_multiple_locations(self): |
| 144 self.tool.executive = MockExecutive2() | 143 self.tool.executive = MockExecutive2() |
| 145 | 144 |
| 146 port = self.tool.port_factory.get('test-win-win7') | 145 port = self.tool.port_factory.get('test-win-win7') |
| 147 self._write( | 146 self._write( |
| 148 port.host.filesystem.join(port.layout_tests_dir(), 'platform/test-wi
n-win7/failures/expected/image-expected.txt'), | 147 port.host.filesystem.join(port.layout_tests_dir(), 'platform/test-wi
n-win7/failures/expected/image-expected.txt'), |
| 149 'original win7 result') | 148 'original win7 result') |
| 150 | 149 |
| 151 oc = OutputCapture() | 150 oc = OutputCapture() |
| 152 try: | 151 try: |
| 153 options = optparse.Values({ | 152 options = optparse.Values({ |
| 154 'builder': "MOCK Win7", | 153 'builder': "MOCK Win7", |
| 155 'suffixes': "txt", | 154 'suffixes': "txt", |
| 156 'verbose': True, | 155 'verbose': True, |
| 157 'test': "failures/expected/image.html", | 156 'test': "failures/expected/image.html", |
| 158 'results_directory': None | 157 'results_directory': None |
| 159 }) | 158 }) |
| 160 oc.capture_output() | 159 oc.capture_output() |
| 161 self.command.execute(options, [], self.tool) | 160 self.command.execute(options, [], self.tool) |
| 162 finally: | 161 finally: |
| 163 out, _, _ = oc.restore_output() | 162 oc.restore_output() |
| 164 | 163 |
| 165 self.assertMultiLineEqual( | 164 self.assertMultiLineEqual( |
| 166 self._read(self.tool.filesystem.join( | 165 self._read(self.tool.filesystem.join( |
| 167 port.layout_tests_dir(), | 166 port.layout_tests_dir(), |
| 168 'platform/test-linux-trusty/failures/expected/image-expected.txt
')), | 167 'platform/test-linux-trusty/failures/expected/image-expected.txt
')), |
| 169 'original win7 result') | 168 'original win7 result') |
| 170 self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join( | 169 self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join( |
| 171 port.layout_tests_dir(), 'platform/test-linux-precise/userscripts/an
other-test-expected.txt'))) | 170 port.layout_tests_dir(), 'platform/test-linux-precise/userscripts/an
other-test-expected.txt'))) |
| 172 self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join( | 171 self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join( |
| 173 port.layout_tests_dir(), 'platform/test-mac-mac10.10/userscripts/ano
ther-test-expected.txt'))) | 172 port.layout_tests_dir(), 'platform/test-mac-mac10.10/userscripts/ano
ther-test-expected.txt'))) |
| 174 self.assertMultiLineEqual(out, '{"add": [], "remove-lines": [], "delete"
: []}\n') | |
| 175 | 173 |
| 176 def test_no_copy_existing_baseline(self): | 174 def test_no_copy_existing_baseline(self): |
| 177 self.tool.executive = MockExecutive2() | 175 self.tool.executive = MockExecutive2() |
| 178 | 176 |
| 179 port = self.tool.port_factory.get('test-win-win7') | 177 port = self.tool.port_factory.get('test-win-win7') |
| 180 self._write( | 178 self._write( |
| 181 port.host.filesystem.join( | 179 port.host.filesystem.join( |
| 182 port.layout_tests_dir(), | 180 port.layout_tests_dir(), |
| 183 'platform/test-win-win7/failures/expected/image-expected.txt'), | 181 'platform/test-win-win7/failures/expected/image-expected.txt'), |
| 184 'original win7 result') | 182 'original win7 result') |
| 185 | 183 |
| 186 oc = OutputCapture() | 184 oc = OutputCapture() |
| 187 try: | 185 try: |
| 188 options = optparse.Values({ | 186 options = optparse.Values({ |
| 189 'builder': "MOCK Win7", | 187 'builder': "MOCK Win7", |
| 190 'suffixes': "txt", | 188 'suffixes': "txt", |
| 191 'verbose': True, | 189 'verbose': True, |
| 192 'test': "failures/expected/image.html", | 190 'test': "failures/expected/image.html", |
| 193 'results_directory': None | 191 'results_directory': None |
| 194 }) | 192 }) |
| 195 oc.capture_output() | 193 oc.capture_output() |
| 196 self.command.execute(options, [], self.tool) | 194 self.command.execute(options, [], self.tool) |
| 197 finally: | 195 finally: |
| 198 out, _, _ = oc.restore_output() | 196 oc.restore_output() |
| 199 | 197 |
| 200 self.assertMultiLineEqual( | 198 self.assertMultiLineEqual( |
| 201 self._read(self.tool.filesystem.join( | 199 self._read(self.tool.filesystem.join( |
| 202 port.layout_tests_dir(), | 200 port.layout_tests_dir(), |
| 203 'platform/test-linux-trusty/failures/expected/image-expected.txt
')), | 201 'platform/test-linux-trusty/failures/expected/image-expected.txt
')), |
| 204 'original win7 result') | 202 'original win7 result') |
| 205 self.assertMultiLineEqual( | 203 self.assertMultiLineEqual( |
| 206 self._read(self.tool.filesystem.join( | 204 self._read(self.tool.filesystem.join( |
| 207 port.layout_tests_dir(), | 205 port.layout_tests_dir(), |
| 208 'platform/test-win-win7/failures/expected/image-expected.txt')), | 206 'platform/test-win-win7/failures/expected/image-expected.txt')), |
| 209 'original win7 result') | 207 'original win7 result') |
| 210 self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join( | 208 self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join( |
| 211 port.layout_tests_dir(), 'platform/test-mac-mac10.10/userscripts/ano
ther-test-expected.txt'))) | 209 port.layout_tests_dir(), 'platform/test-mac-mac10.10/userscripts/ano
ther-test-expected.txt'))) |
| 212 self.assertMultiLineEqual(out, '{"add": [], "remove-lines": [], "delete"
: []}\n') | |
| 213 | 210 |
| 214 def test_no_copy_skipped_test(self): | 211 def test_no_copy_skipped_test(self): |
| 215 self.tool.executive = MockExecutive2() | 212 self.tool.executive = MockExecutive2() |
| 216 port = self.tool.port_factory.get('test-win-win7') | 213 port = self.tool.port_factory.get('test-win-win7') |
| 217 fs = self.tool.filesystem | 214 fs = self.tool.filesystem |
| 218 self._write( | 215 self._write( |
| 219 fs.join( | 216 fs.join( |
| 220 port.layout_tests_dir(), | 217 port.layout_tests_dir(), |
| 221 'platform/test-win-win7/failures/expected/image-expected.txt'), | 218 'platform/test-win-win7/failures/expected/image-expected.txt'), |
| 222 'original win7 result') | 219 'original win7 result') |
| (...skipping 93 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 316 "bug(z) [ Linux ] userscripts/another-test.html [ Failure ]\n")) | 313 "bug(z) [ Linux ] userscripts/another-test.html [ Failure ]\n")) |
| 317 self.command._rebaseline_test_and_update_expectations(self.options(resul
ts_directory='/tmp')) | 314 self.command._rebaseline_test_and_update_expectations(self.options(resul
ts_directory='/tmp')) |
| 318 self.assertItemsEqual(self.tool.web.urls_fetched, ['file:///tmp/userscri
pts/another-test-actual.txt']) | 315 self.assertItemsEqual(self.tool.web.urls_fetched, ['file:///tmp/userscri
pts/another-test-actual.txt']) |
| 319 | 316 |
| 320 def test_rebaseline_reftest(self): | 317 def test_rebaseline_reftest(self): |
| 321 self._write("userscripts/another-test.html", "test data") | 318 self._write("userscripts/another-test.html", "test data") |
| 322 self._write("userscripts/another-test-expected.html", "generic result") | 319 self._write("userscripts/another-test-expected.html", "generic result") |
| 323 OutputCapture().assert_outputs( | 320 OutputCapture().assert_outputs( |
| 324 self, self.command._rebaseline_test_and_update_expectations, args=[s
elf.options(suffixes='png')], | 321 self, self.command._rebaseline_test_and_update_expectations, args=[s
elf.options(suffixes='png')], |
| 325 expected_logs="Cannot rebaseline image result for reftest: userscrip
ts/another-test.html\n") | 322 expected_logs="Cannot rebaseline image result for reftest: userscrip
ts/another-test.html\n") |
| 326 self.assertDictEqual(self.command._scm_changes.to_dict(), {'add': [], 'r
emove-lines': [], "delete": []}) | 323 self.assertDictEqual(self.command.expectation_line_changes.to_dict(), {'
remove-lines': []}) |
| 327 | |
| 328 def test_rebaseline_test_and_print_scm_changes(self): | |
| 329 self.command._print_scm_changes = True | |
| 330 self.command._scm_changes = ChangeSet() | |
| 331 self.tool._scm.exists = lambda x: False | |
| 332 | |
| 333 self.command._rebaseline_test("MOCK Trusty", "userscripts/another-test.h
tml", "txt", None) | |
| 334 | |
| 335 self.assertDictEqual( | |
| 336 self.command._scm_changes.to_dict(), | |
| 337 { | |
| 338 'add': ['/test.checkout/LayoutTests/platform/test-linux-trusty/u
serscripts/another-test-expected.txt'], | |
| 339 'delete': [], | |
| 340 'remove-lines': [] | |
| 341 }) | |
| 342 | 324 |
| 343 def test_rebaseline_test_internal_with_port_that_lacks_buildbot(self): | 325 def test_rebaseline_test_internal_with_port_that_lacks_buildbot(self): |
| 344 self.tool.executive = MockExecutive2() | 326 self.tool.executive = MockExecutive2() |
| 345 | 327 |
| 346 port = self.tool.port_factory.get('test-win-win7') | 328 port = self.tool.port_factory.get('test-win-win7') |
| 347 self._write( | 329 self._write( |
| 348 port.host.filesystem.join( | 330 port.host.filesystem.join( |
| 349 port.layout_tests_dir(), | 331 port.layout_tests_dir(), |
| 350 'platform/test-win-win10/failures/expected/image-expected.txt'), | 332 'platform/test-win-win10/failures/expected/image-expected.txt'), |
| 351 'original win10 result') | 333 'original win10 result') |
| (...skipping 15 matching lines...) Expand all Loading... |
| 367 out, _, _ = oc.restore_output() | 349 out, _, _ = oc.restore_output() |
| 368 | 350 |
| 369 self.assertMultiLineEqual( | 351 self.assertMultiLineEqual( |
| 370 self._read(self.tool.filesystem.join( | 352 self._read(self.tool.filesystem.join( |
| 371 port.layout_tests_dir(), | 353 port.layout_tests_dir(), |
| 372 'platform/test-win-win10/failures/expected/image-expected.txt'))
, | 354 'platform/test-win-win10/failures/expected/image-expected.txt'))
, |
| 373 'MOCK Web result, convert 404 to None=True') | 355 'MOCK Web result, convert 404 to None=True') |
| 374 self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join( | 356 self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join( |
| 375 port.layout_tests_dir(), 'platform/test-win-win7/failures/expected/i
mage-expected.txt'))) | 357 port.layout_tests_dir(), 'platform/test-win-win7/failures/expected/i
mage-expected.txt'))) |
| 376 self.assertMultiLineEqual( | 358 self.assertMultiLineEqual( |
| 377 out, '{"add": [], "remove-lines": [{"test": "failures/expected/image
.html", "builder": "MOCK Win10"}], "delete": []}\n') | 359 out, '{"remove-lines": [{"test": "failures/expected/image.html", "bu
ilder": "MOCK Win10"}]}\n') |
| 378 | 360 |
| 379 | 361 |
| 380 class TestAbstractParallelRebaselineCommand(BaseTestCase): | 362 class TestAbstractParallelRebaselineCommand(BaseTestCase): |
| 381 command_constructor = AbstractParallelRebaselineCommand | 363 command_constructor = AbstractParallelRebaselineCommand |
| 382 | 364 |
| 383 def test_builders_to_fetch_from(self): | 365 def test_builders_to_fetch_from(self): |
| 384 builders_to_fetch = self.command._builders_to_fetch_from( | 366 builders_to_fetch = self.command._builders_to_fetch_from( |
| 385 ["MOCK Win10", "MOCK Win7 (dbg)(1)", "MOCK Win7 (dbg)(2)", "MOCK Win
7"]) | 367 ["MOCK Win10", "MOCK Win7 (dbg)(1)", "MOCK Win7 (dbg)(2)", "MOCK Win
7"]) |
| 386 self.assertEqual(builders_to_fetch, ["MOCK Win7", "MOCK Win10"]) | 368 self.assertEqual(builders_to_fetch, ["MOCK Win7", "MOCK Win10"]) |
| 387 | 369 |
| (...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 431 self.command.rebaseline(self.options(), {"userscripts/first-test.html":
{Build("MOCK Win7"): ["txt", "png"]}}) | 413 self.command.rebaseline(self.options(), {"userscripts/first-test.html":
{Build("MOCK Win7"): ["txt", "png"]}}) |
| 432 | 414 |
| 433 # Note that we have one run_in_parallel() call followed by a run_command
() | 415 # Note that we have one run_in_parallel() call followed by a run_command
() |
| 434 self.assertEqual( | 416 self.assertEqual( |
| 435 self.tool.executive.calls, | 417 self.tool.executive.calls, |
| 436 [ | 418 [ |
| 437 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt,png', | 419 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt,png', |
| 438 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--verbose']], | 420 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--verbose']], |
| 439 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 't
xt,png', | 421 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 't
xt,png', |
| 440 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--verbose']], | 422 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--verbose']], |
| 441 [['python', 'echo', 'optimize-baselines', '--no-modify-scm', '--
suffixes', 'txt,png', | 423 [['python', 'echo', 'optimize-baselines', '--suffixes', 'txt,png
', |
| 442 'userscripts/first-test.html', '--verbose']] | 424 'userscripts/first-test.html', '--verbose']] |
| 443 ]) | 425 ]) |
| 444 | 426 |
| 445 def test_rebaseline_debug(self): | 427 def test_rebaseline_debug(self): |
| 446 self._setup_mock_build_data() | 428 self._setup_mock_build_data() |
| 447 | 429 |
| 448 self._write("userscripts/first-test.html", "Dummy test contents") | 430 self._write("userscripts/first-test.html", "Dummy test contents") |
| 449 self.command.rebaseline(self.options(), {"userscripts/first-test.html":
{Build("MOCK Win7 (dbg)"): ["txt", "png"]}}) | 431 self.command.rebaseline(self.options(), {"userscripts/first-test.html":
{Build("MOCK Win7 (dbg)"): ["txt", "png"]}}) |
| 450 | 432 |
| 451 # Note that we have one run_in_parallel() call followed by a run_command
() | 433 # Note that we have one run_in_parallel() call followed by a run_command
() |
| 452 self.assertEqual( | 434 self.assertEqual( |
| 453 self.tool.executive.calls, | 435 self.tool.executive.calls, |
| 454 [ | 436 [ |
| 455 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt,png', | 437 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt,png', |
| 456 '--builder', 'MOCK Win7 (dbg)', '--test', 'userscripts/first-t
est.html', '--verbose']], | 438 '--builder', 'MOCK Win7 (dbg)', '--test', 'userscripts/first-t
est.html', '--verbose']], |
| 457 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 't
xt,png', '--builder', | 439 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 't
xt,png', '--builder', |
| 458 'MOCK Win7 (dbg)', '--test', 'userscripts/first-test.html', '-
-verbose']], | 440 'MOCK Win7 (dbg)', '--test', 'userscripts/first-test.html', '-
-verbose']], |
| 459 [['python', 'echo', 'optimize-baselines', '--no-modify-scm', '--
suffixes', 'txt,png', | 441 [['python', 'echo', 'optimize-baselines', '--suffixes', 'txt,png
', |
| 460 'userscripts/first-test.html', '--verbose']] | 442 'userscripts/first-test.html', '--verbose']] |
| 461 ]) | 443 ]) |
| 462 | 444 |
| 463 def test_no_optimize(self): | 445 def test_no_optimize(self): |
| 464 self._setup_mock_build_data() | 446 self._setup_mock_build_data() |
| 465 self._write("userscripts/first-test.html", "Dummy test contents") | 447 self._write("userscripts/first-test.html", "Dummy test contents") |
| 466 self.command.rebaseline( | 448 self.command.rebaseline( |
| 467 self.options(optimize=False), | 449 self.options(optimize=False), |
| 468 {"userscripts/first-test.html": {Build("MOCK Win7"): ["txt", "png"]}
}) | 450 {"userscripts/first-test.html": {Build("MOCK Win7"): ["txt", "png"]}
}) |
| 469 | 451 |
| (...skipping 459 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 929 class MockLineRemovingExecutive(MockExecutive): | 911 class MockLineRemovingExecutive(MockExecutive): |
| 930 | 912 |
| 931 def run_in_parallel(self, commands): | 913 def run_in_parallel(self, commands): |
| 932 assert len(commands) | 914 assert len(commands) |
| 933 | 915 |
| 934 num_previous_calls = len(self.calls) | 916 num_previous_calls = len(self.calls) |
| 935 command_outputs = [] | 917 command_outputs = [] |
| 936 for cmd_line, cwd in commands: | 918 for cmd_line, cwd in commands: |
| 937 out = self.run_command(cmd_line, cwd=cwd) | 919 out = self.run_command(cmd_line, cwd=cwd) |
| 938 if 'rebaseline-test-internal' in cmd_line: | 920 if 'rebaseline-test-internal' in cmd_line: |
| 939 out = '{"add": [], "remove-lines": [{"test": "%s", "builder": "%
s"}], "delete": []}\n' % (cmd_line[8], cmd_line[6]) | 921 out = '{"remove-lines": [{"test": "%s", "builder": "%s"}]}\n' %
(cmd_line[8], cmd_line[6]) |
| 940 command_outputs.append([0, out, '']) | 922 command_outputs.append([0, out, '']) |
| 941 | 923 |
| 942 new_calls = self.calls[num_previous_calls:] | 924 new_calls = self.calls[num_previous_calls:] |
| 943 self.calls = self.calls[:num_previous_calls] | 925 self.calls = self.calls[:num_previous_calls] |
| 944 self.calls.append(new_calls) | 926 self.calls.append(new_calls) |
| 945 return command_outputs | 927 return command_outputs |
| OLD | NEW |