| OLD | NEW |
| 1 # Copyright 2016 The Chromium Authors. All rights reserved. | 1 # Copyright 2016 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 import optparse | 5 import optparse |
| 6 import unittest | 6 import unittest |
| 7 | 7 |
| 8 from webkitpy.common.net.buildbot import Build | 8 from webkitpy.common.net.buildbot import Build |
| 9 from webkitpy.common.net.layouttestresults import LayoutTestResults | 9 from webkitpy.common.net.layouttestresults import LayoutTestResults |
| 10 from webkitpy.common.system.executive_mock import MockExecutive, MockExecutive2 | 10 from webkitpy.common.system.executive_mock import MockExecutive, MockExecutive2 |
| 11 from webkitpy.common.system.outputcapture import OutputCapture | 11 from webkitpy.common.system.outputcapture import OutputCapture |
| 12 from webkitpy.layout_tests.builder_list import BuilderList | 12 from webkitpy.layout_tests.builder_list import BuilderList |
| 13 from webkitpy.tool.commands.rebaseline import ( | 13 from webkitpy.tool.commands.rebaseline import ( |
| 14 AbstractParallelRebaselineCommand, CopyExistingBaselinesInternal, | 14 AbstractParallelRebaselineCommand, CopyExistingBaselinesInternal, |
| 15 Rebaseline, RebaselineExpectations, RebaselineJson, RebaselineTest, ChangeSe
t | 15 Rebaseline, RebaselineExpectations, RebaselineJson, RebaselineTest |
| 16 ) | 16 ) |
| 17 from webkitpy.tool.mock_tool import MockWebKitPatch | 17 from webkitpy.tool.mock_tool import MockWebKitPatch |
| 18 | 18 |
| 19 | 19 |
| 20 # pylint: disable=protected-access | 20 # pylint: disable=protected-access |
| 21 class BaseTestCase(unittest.TestCase): | 21 class BaseTestCase(unittest.TestCase): |
| 22 MOCK_WEB_RESULT = 'MOCK Web result, convert 404 to None=True' | 22 MOCK_WEB_RESULT = 'MOCK Web result, convert 404 to None=True' |
| 23 WEB_PREFIX = 'https://storage.googleapis.com/chromium-layout-test-archives/M
OCK_Mac10_11/results/layout-test-results' | 23 WEB_PREFIX = 'https://storage.googleapis.com/chromium-layout-test-archives/M
OCK_Mac10_11/results/layout-test-results' |
| 24 | 24 |
| 25 command_constructor = None | 25 command_constructor = None |
| (...skipping 116 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 142 self._read(self.tool.filesystem.join( | 142 self._read(self.tool.filesystem.join( |
| 143 port.layout_tests_dir(), | 143 port.layout_tests_dir(), |
| 144 'platform/test-mac-mac10.11/failures/expected/image-expected.txt
')), | 144 'platform/test-mac-mac10.11/failures/expected/image-expected.txt
')), |
| 145 'original mac10.11 result') | 145 'original mac10.11 result') |
| 146 self.assertEqual( | 146 self.assertEqual( |
| 147 self._read(self.tool.filesystem.join( | 147 self._read(self.tool.filesystem.join( |
| 148 port.layout_tests_dir(), | 148 port.layout_tests_dir(), |
| 149 'platform/test-mac-mac10.10/failures/expected/image-expected.txt
')), | 149 'platform/test-mac-mac10.10/failures/expected/image-expected.txt
')), |
| 150 'original mac10.11 result') | 150 'original mac10.11 result') |
| 151 | 151 |
| 152 def test_copying_overwritten_baseline_to_multiple_locations(self): |
| 153 self.tool.executive = MockExecutive2() |
| 154 |
| 152 def test_copy_baseline_win7_to_linux_trusty(self): | 155 def test_copy_baseline_win7_to_linux_trusty(self): |
| 153 port = self.tool.port_factory.get('test-win-win7') | 156 port = self.tool.port_factory.get('test-win-win7') |
| 154 self._write( | 157 self._write( |
| 155 self.tool.filesystem.join( | 158 self.tool.filesystem.join( |
| 156 port.layout_tests_dir(), | 159 port.layout_tests_dir(), |
| 157 'platform/test-win-win7/failures/expected/image-expected.txt'), | 160 'platform/test-win-win7/failures/expected/image-expected.txt'), |
| 158 'original win7 result') | 161 'original win7 result') |
| 159 self.assertFalse(self.tool.filesystem.exists( | 162 self.assertFalse(self.tool.filesystem.exists( |
| 160 self.tool.filesystem.join( | 163 self.tool.filesystem.join( |
| 161 port.layout_tests_dir(), | 164 port.layout_tests_dir(), |
| (...skipping 124 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 286 "bug(z) [ Linux ] userscripts/another-test.html [ Failure ]\n")) | 289 "bug(z) [ Linux ] userscripts/another-test.html [ Failure ]\n")) |
| 287 self.command._rebaseline_test_and_update_expectations(self.options(resul
ts_directory='/tmp')) | 290 self.command._rebaseline_test_and_update_expectations(self.options(resul
ts_directory='/tmp')) |
| 288 self.assertItemsEqual(self.tool.web.urls_fetched, ['file:///tmp/userscri
pts/another-test-actual.txt']) | 291 self.assertItemsEqual(self.tool.web.urls_fetched, ['file:///tmp/userscri
pts/another-test-actual.txt']) |
| 289 | 292 |
| 290 def test_rebaseline_reftest(self): | 293 def test_rebaseline_reftest(self): |
| 291 self._write("userscripts/another-test.html", "test data") | 294 self._write("userscripts/another-test.html", "test data") |
| 292 self._write("userscripts/another-test-expected.html", "generic result") | 295 self._write("userscripts/another-test-expected.html", "generic result") |
| 293 OutputCapture().assert_outputs( | 296 OutputCapture().assert_outputs( |
| 294 self, self.command._rebaseline_test_and_update_expectations, args=[s
elf.options(suffixes='png')], | 297 self, self.command._rebaseline_test_and_update_expectations, args=[s
elf.options(suffixes='png')], |
| 295 expected_logs="Cannot rebaseline image result for reftest: userscrip
ts/another-test.html\n") | 298 expected_logs="Cannot rebaseline image result for reftest: userscrip
ts/another-test.html\n") |
| 296 self.assertDictEqual(self.command._scm_changes.to_dict(), {'add': [], 'r
emove-lines': [], "delete": []}) | 299 self.assertDictEqual(self.command.expectation_line_changes.to_dict(), {'
remove-lines': []}) |
| 297 | |
| 298 def test_rebaseline_test_and_print_scm_changes(self): | |
| 299 self.command._print_scm_changes = True | |
| 300 self.command._scm_changes = ChangeSet() | |
| 301 self.tool._scm.exists = lambda x: False | |
| 302 | |
| 303 self.command._rebaseline_test("MOCK Trusty", "userscripts/another-test.h
tml", "txt", None) | |
| 304 | |
| 305 self.assertDictEqual( | |
| 306 self.command._scm_changes.to_dict(), | |
| 307 { | |
| 308 'add': ['/test.checkout/LayoutTests/platform/test-linux-trusty/u
serscripts/another-test-expected.txt'], | |
| 309 'delete': [], | |
| 310 'remove-lines': [] | |
| 311 }) | |
| 312 | 300 |
| 313 def test_rebaseline_test_internal_with_port_that_lacks_buildbot(self): | 301 def test_rebaseline_test_internal_with_port_that_lacks_buildbot(self): |
| 314 self.tool.executive = MockExecutive2() | 302 self.tool.executive = MockExecutive2() |
| 315 | 303 |
| 316 port = self.tool.port_factory.get('test-win-win7') | 304 port = self.tool.port_factory.get('test-win-win7') |
| 317 self._write( | 305 self._write( |
| 318 port.host.filesystem.join( | 306 port.host.filesystem.join( |
| 319 port.layout_tests_dir(), | 307 port.layout_tests_dir(), |
| 320 'platform/test-win-win10/failures/expected/image-expected.txt'), | 308 'platform/test-win-win10/failures/expected/image-expected.txt'), |
| 321 'original win10 result') | 309 'original win10 result') |
| (...skipping 15 matching lines...) Expand all Loading... |
| 337 out, _, _ = oc.restore_output() | 325 out, _, _ = oc.restore_output() |
| 338 | 326 |
| 339 self.assertMultiLineEqual( | 327 self.assertMultiLineEqual( |
| 340 self._read(self.tool.filesystem.join( | 328 self._read(self.tool.filesystem.join( |
| 341 port.layout_tests_dir(), | 329 port.layout_tests_dir(), |
| 342 'platform/test-win-win10/failures/expected/image-expected.txt'))
, | 330 'platform/test-win-win10/failures/expected/image-expected.txt'))
, |
| 343 'MOCK Web result, convert 404 to None=True') | 331 'MOCK Web result, convert 404 to None=True') |
| 344 self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join( | 332 self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join( |
| 345 port.layout_tests_dir(), 'platform/test-win-win7/failures/expected/i
mage-expected.txt'))) | 333 port.layout_tests_dir(), 'platform/test-win-win7/failures/expected/i
mage-expected.txt'))) |
| 346 self.assertMultiLineEqual( | 334 self.assertMultiLineEqual( |
| 347 out, '{"add": [], "remove-lines": [{"test": "failures/expected/image
.html", "builder": "MOCK Win10"}], "delete": []}\n') | 335 out, '{"remove-lines": [{"test": "failures/expected/image.html", "bu
ilder": "MOCK Win10"}]}\n') |
| 348 | 336 |
| 349 | 337 |
| 350 class TestAbstractParallelRebaselineCommand(BaseTestCase): | 338 class TestAbstractParallelRebaselineCommand(BaseTestCase): |
| 351 command_constructor = AbstractParallelRebaselineCommand | 339 command_constructor = AbstractParallelRebaselineCommand |
| 352 | 340 |
| 353 def test_builders_to_fetch_from(self): | 341 def test_builders_to_fetch_from(self): |
| 354 builders_to_fetch = self.command._builders_to_fetch_from( | 342 builders_to_fetch = self.command._builders_to_fetch_from( |
| 355 ["MOCK Win10", "MOCK Win7 (dbg)(1)", "MOCK Win7 (dbg)(2)", "MOCK Win
7"]) | 343 ["MOCK Win10", "MOCK Win7 (dbg)(1)", "MOCK Win7 (dbg)(2)", "MOCK Win
7"]) |
| 356 self.assertEqual(builders_to_fetch, ["MOCK Win7", "MOCK Win10"]) | 344 self.assertEqual(builders_to_fetch, ["MOCK Win7", "MOCK Win10"]) |
| 357 | 345 |
| (...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 401 self.command.rebaseline(self.options(), {"userscripts/first-test.html":
{Build("MOCK Win7"): ["txt", "png"]}}) | 389 self.command.rebaseline(self.options(), {"userscripts/first-test.html":
{Build("MOCK Win7"): ["txt", "png"]}}) |
| 402 | 390 |
| 403 # Note that we have one run_in_parallel() call followed by a run_command
() | 391 # Note that we have one run_in_parallel() call followed by a run_command
() |
| 404 self.assertEqual( | 392 self.assertEqual( |
| 405 self.tool.executive.calls, | 393 self.tool.executive.calls, |
| 406 [ | 394 [ |
| 407 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt,png', | 395 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt,png', |
| 408 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--verbose']], | 396 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--verbose']], |
| 409 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 't
xt,png', | 397 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 't
xt,png', |
| 410 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--verbose']], | 398 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--verbose']], |
| 411 [['python', 'echo', 'optimize-baselines', '--no-modify-scm', '--
suffixes', 'txt,png', | 399 [['python', 'echo', 'optimize-baselines', '--suffixes', 'txt,png
', |
| 412 'userscripts/first-test.html', '--verbose']] | 400 'userscripts/first-test.html', '--verbose']] |
| 413 ]) | 401 ]) |
| 414 | 402 |
| 415 def test_rebaseline_debug(self): | 403 def test_rebaseline_debug(self): |
| 416 self._setup_mock_build_data() | 404 self._setup_mock_build_data() |
| 417 | 405 |
| 418 self._write("userscripts/first-test.html", "Dummy test contents") | 406 self._write("userscripts/first-test.html", "Dummy test contents") |
| 419 self.command.rebaseline(self.options(), {"userscripts/first-test.html":
{Build("MOCK Win7 (dbg)"): ["txt", "png"]}}) | 407 self.command.rebaseline(self.options(), {"userscripts/first-test.html":
{Build("MOCK Win7 (dbg)"): ["txt", "png"]}}) |
| 420 | 408 |
| 421 # Note that we have one run_in_parallel() call followed by a run_command
() | 409 # Note that we have one run_in_parallel() call followed by a run_command
() |
| 422 self.assertEqual( | 410 self.assertEqual( |
| 423 self.tool.executive.calls, | 411 self.tool.executive.calls, |
| 424 [ | 412 [ |
| 425 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt,png', | 413 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt,png', |
| 426 '--builder', 'MOCK Win7 (dbg)', '--test', 'userscripts/first-t
est.html', '--verbose']], | 414 '--builder', 'MOCK Win7 (dbg)', '--test', 'userscripts/first-t
est.html', '--verbose']], |
| 427 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 't
xt,png', '--builder', | 415 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 't
xt,png', '--builder', |
| 428 'MOCK Win7 (dbg)', '--test', 'userscripts/first-test.html', '-
-verbose']], | 416 'MOCK Win7 (dbg)', '--test', 'userscripts/first-test.html', '-
-verbose']], |
| 429 [['python', 'echo', 'optimize-baselines', '--no-modify-scm', '--
suffixes', 'txt,png', | 417 [['python', 'echo', 'optimize-baselines', '--suffixes', 'txt,png
', |
| 430 'userscripts/first-test.html', '--verbose']] | 418 'userscripts/first-test.html', '--verbose']] |
| 431 ]) | 419 ]) |
| 432 | 420 |
| 433 def test_no_optimize(self): | 421 def test_no_optimize(self): |
| 434 self._setup_mock_build_data() | 422 self._setup_mock_build_data() |
| 435 self._write("userscripts/first-test.html", "Dummy test contents") | 423 self._write("userscripts/first-test.html", "Dummy test contents") |
| 436 self.command.rebaseline( | 424 self.command.rebaseline( |
| 437 self.options(optimize=False), | 425 self.options(optimize=False), |
| 438 {"userscripts/first-test.html": {Build("MOCK Win7"): ["txt", "png"]}
}) | 426 {"userscripts/first-test.html": {Build("MOCK Win7"): ["txt", "png"]}
}) |
| 439 | 427 |
| (...skipping 459 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 899 class MockLineRemovingExecutive(MockExecutive): | 887 class MockLineRemovingExecutive(MockExecutive): |
| 900 | 888 |
| 901 def run_in_parallel(self, commands): | 889 def run_in_parallel(self, commands): |
| 902 assert len(commands) | 890 assert len(commands) |
| 903 | 891 |
| 904 num_previous_calls = len(self.calls) | 892 num_previous_calls = len(self.calls) |
| 905 command_outputs = [] | 893 command_outputs = [] |
| 906 for cmd_line, cwd in commands: | 894 for cmd_line, cwd in commands: |
| 907 out = self.run_command(cmd_line, cwd=cwd) | 895 out = self.run_command(cmd_line, cwd=cwd) |
| 908 if 'rebaseline-test-internal' in cmd_line: | 896 if 'rebaseline-test-internal' in cmd_line: |
| 909 out = '{"add": [], "remove-lines": [{"test": "%s", "builder": "%
s"}], "delete": []}\n' % (cmd_line[8], cmd_line[6]) | 897 out = '{"remove-lines": [{"test": "%s", "builder": "%s"}]}\n' %
(cmd_line[8], cmd_line[6]) |
| 910 command_outputs.append([0, out, '']) | 898 command_outputs.append([0, out, '']) |
| 911 | 899 |
| 912 new_calls = self.calls[num_previous_calls:] | 900 new_calls = self.calls[num_previous_calls:] |
| 913 self.calls = self.calls[:num_previous_calls] | 901 self.calls = self.calls[:num_previous_calls] |
| 914 self.calls.append(new_calls) | 902 self.calls.append(new_calls) |
| 915 return command_outputs | 903 return command_outputs |
| OLD | NEW |