| OLD | NEW |
| 1 # Copyright 2016 The Chromium Authors. All rights reserved. | 1 # Copyright 2016 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 import optparse | 5 import optparse |
| 6 import unittest | 6 import unittest |
| 7 | 7 |
| 8 from webkitpy.common.net.buildbot import Build | 8 from webkitpy.common.net.buildbot import Build |
| 9 from webkitpy.common.net.layouttestresults import LayoutTestResults | 9 from webkitpy.common.net.layouttestresults import LayoutTestResults |
| 10 from webkitpy.common.system.executive_mock import MockExecutive, MockExecutive2 | 10 from webkitpy.common.system.executive_mock import MockExecutive, MockExecutive2 |
| (...skipping 393 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 404 "first-test.html": { | 404 "first-test.html": { |
| 405 "expected": "NEEDSREBASELINE", | 405 "expected": "NEEDSREBASELINE", |
| 406 "actual": "PASS" | 406 "actual": "PASS" |
| 407 } | 407 } |
| 408 } | 408 } |
| 409 } | 409 } |
| 410 })) | 410 })) |
| 411 | 411 |
| 412 self._write(self.mac_expectations_path, "Bug(x) userscripts/first-test.h
tml [ Failure ]\n") | 412 self._write(self.mac_expectations_path, "Bug(x) userscripts/first-test.h
tml [ Failure ]\n") |
| 413 self._write("userscripts/first-test.html", "Dummy test contents") | 413 self._write("userscripts/first-test.html", "Dummy test contents") |
| 414 self.command._rebaseline(self.options(), {"userscripts/first-test.html":
{Build("MOCK Win7"): ["txt", "png"]}}) | 414 self.command.rebaseline(self.options(), {"userscripts/first-test.html":
{Build("MOCK Win7"): ["txt", "png"]}}) |
| 415 | 415 |
| 416 self.assertEqual(self.tool.executive.calls, []) | 416 self.assertEqual(self.tool.executive.calls, []) |
| 417 | 417 |
| 418 def test_rebaseline_all(self): | 418 def test_rebaseline_all(self): |
| 419 self._setup_mock_build_data() | 419 self._setup_mock_build_data() |
| 420 | 420 |
| 421 self._write("userscripts/first-test.html", "Dummy test contents") | 421 self._write("userscripts/first-test.html", "Dummy test contents") |
| 422 self.command._rebaseline(self.options(), {"userscripts/first-test.html":
{Build("MOCK Win7"): ["txt", "png"]}}) | 422 self.command.rebaseline(self.options(), {"userscripts/first-test.html":
{Build("MOCK Win7"): ["txt", "png"]}}) |
| 423 | 423 |
| 424 # Note that we have one run_in_parallel() call followed by a run_command
() | 424 # Note that we have one run_in_parallel() call followed by a run_command
() |
| 425 self.assertEqual( | 425 self.assertEqual( |
| 426 self.tool.executive.calls, | 426 self.tool.executive.calls, |
| 427 [ | 427 [ |
| 428 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt,png', | 428 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt,png', |
| 429 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--verbose']], | 429 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--verbose']], |
| 430 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 't
xt,png', | 430 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 't
xt,png', |
| 431 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--verbose']], | 431 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--verbose']], |
| 432 [['python', 'echo', 'optimize-baselines', '--no-modify-scm', '--
suffixes', 'txt,png', | 432 [['python', 'echo', 'optimize-baselines', '--no-modify-scm', '--
suffixes', 'txt,png', |
| 433 'userscripts/first-test.html', '--verbose']] | 433 'userscripts/first-test.html', '--verbose']] |
| 434 ]) | 434 ]) |
| 435 | 435 |
| 436 def test_rebaseline_debug(self): | 436 def test_rebaseline_debug(self): |
| 437 self._setup_mock_build_data() | 437 self._setup_mock_build_data() |
| 438 | 438 |
| 439 self._write("userscripts/first-test.html", "Dummy test contents") | 439 self._write("userscripts/first-test.html", "Dummy test contents") |
| 440 self.command._rebaseline(self.options(), {"userscripts/first-test.html":
{Build("MOCK Win7 (dbg)"): ["txt", "png"]}}) | 440 self.command.rebaseline(self.options(), {"userscripts/first-test.html":
{Build("MOCK Win7 (dbg)"): ["txt", "png"]}}) |
| 441 | 441 |
| 442 # Note that we have one run_in_parallel() call followed by a run_command
() | 442 # Note that we have one run_in_parallel() call followed by a run_command
() |
| 443 self.assertEqual( | 443 self.assertEqual( |
| 444 self.tool.executive.calls, | 444 self.tool.executive.calls, |
| 445 [ | 445 [ |
| 446 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt,png', | 446 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt,png', |
| 447 '--builder', 'MOCK Win7 (dbg)', '--test', 'userscripts/first-t
est.html', '--verbose']], | 447 '--builder', 'MOCK Win7 (dbg)', '--test', 'userscripts/first-t
est.html', '--verbose']], |
| 448 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 't
xt,png', '--builder', | 448 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 't
xt,png', '--builder', |
| 449 'MOCK Win7 (dbg)', '--test', 'userscripts/first-test.html', '-
-verbose']], | 449 'MOCK Win7 (dbg)', '--test', 'userscripts/first-test.html', '-
-verbose']], |
| 450 [['python', 'echo', 'optimize-baselines', '--no-modify-scm', '--
suffixes', 'txt,png', | 450 [['python', 'echo', 'optimize-baselines', '--no-modify-scm', '--
suffixes', 'txt,png', |
| 451 'userscripts/first-test.html', '--verbose']] | 451 'userscripts/first-test.html', '--verbose']] |
| 452 ]) | 452 ]) |
| 453 | 453 |
| 454 def test_no_optimize(self): | 454 def test_no_optimize(self): |
| 455 self._setup_mock_build_data() | 455 self._setup_mock_build_data() |
| 456 self._write("userscripts/first-test.html", "Dummy test contents") | 456 self._write("userscripts/first-test.html", "Dummy test contents") |
| 457 self.command._rebaseline( | 457 self.command.rebaseline( |
| 458 self.options(optimize=False), | 458 self.options(optimize=False), |
| 459 {"userscripts/first-test.html": {Build("MOCK Win7"): ["txt", "png"]}
}) | 459 {"userscripts/first-test.html": {Build("MOCK Win7"): ["txt", "png"]}
}) |
| 460 | 460 |
| 461 # Note that we have only one run_in_parallel() call | 461 # Note that we have only one run_in_parallel() call |
| 462 self.assertEqual( | 462 self.assertEqual( |
| 463 self.tool.executive.calls, | 463 self.tool.executive.calls, |
| 464 [ | 464 [ |
| 465 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt,png', | 465 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt,png', |
| 466 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--verbose']], | 466 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--verbose']], |
| 467 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 't
xt,png', | 467 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 't
xt,png', |
| 468 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--verbose']] | 468 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--verbose']] |
| 469 ]) | 469 ]) |
| 470 | 470 |
| 471 def test_results_directory(self): | 471 def test_results_directory(self): |
| 472 self._setup_mock_build_data() | 472 self._setup_mock_build_data() |
| 473 self._write("userscripts/first-test.html", "Dummy test contents") | 473 self._write("userscripts/first-test.html", "Dummy test contents") |
| 474 self.command._rebaseline( | 474 self.command.rebaseline( |
| 475 self.options(optimize=False, results_directory='/tmp'), | 475 self.options(optimize=False, results_directory='/tmp'), |
| 476 {"userscripts/first-test.html": {Build("MOCK Win7"): ["txt", "png"]}
}) | 476 {"userscripts/first-test.html": {Build("MOCK Win7"): ["txt", "png"]}
}) |
| 477 | 477 |
| 478 # Note that we have only one run_in_parallel() call | 478 # Note that we have only one run_in_parallel() call |
| 479 self.assertEqual( | 479 self.assertEqual( |
| 480 self.tool.executive.calls, | 480 self.tool.executive.calls, |
| 481 [ | 481 [ |
| 482 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt,png', | 482 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt,png', |
| 483 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--results-directory', '/tmp', '--verbose']], | 483 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--results-directory', '/tmp', '--verbose']], |
| 484 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 't
xt,png', | 484 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 't
xt,png', |
| (...skipping 21 matching lines...) Expand all Loading... |
| 506 }) | 506 }) |
| 507 | 507 |
| 508 def test_rebaseline_updates_expectations_file(self): | 508 def test_rebaseline_updates_expectations_file(self): |
| 509 self._write( | 509 self._write( |
| 510 self.mac_expectations_path, | 510 self.mac_expectations_path, |
| 511 ("Bug(x) [ Mac ] userscripts/first-test.html [ Failure ]\n" | 511 ("Bug(x) [ Mac ] userscripts/first-test.html [ Failure ]\n" |
| 512 "bug(z) [ Linux ] userscripts/first-test.html [ Failure ]\n")) | 512 "bug(z) [ Linux ] userscripts/first-test.html [ Failure ]\n")) |
| 513 self._write("userscripts/first-test.html", "Dummy test contents") | 513 self._write("userscripts/first-test.html", "Dummy test contents") |
| 514 self._setup_mock_build_data() | 514 self._setup_mock_build_data() |
| 515 | 515 |
| 516 self.command._rebaseline( | 516 self.command.rebaseline( |
| 517 self.options(), | 517 self.options(), |
| 518 {"userscripts/first-test.html": {Build("MOCK Mac10.11"): ["txt", "pn
g"]}}) | 518 {"userscripts/first-test.html": {Build("MOCK Mac10.11"): ["txt", "pn
g"]}}) |
| 519 | 519 |
| 520 new_expectations = self._read(self.mac_expectations_path) | 520 new_expectations = self._read(self.mac_expectations_path) |
| 521 self.assertMultiLineEqual( | 521 self.assertMultiLineEqual( |
| 522 new_expectations, | 522 new_expectations, |
| 523 ("Bug(x) [ Mac10.10 ] userscripts/first-test.html [ Failure ]\n" | 523 ("Bug(x) [ Mac10.10 ] userscripts/first-test.html [ Failure ]\n" |
| 524 "bug(z) [ Linux ] userscripts/first-test.html [ Failure ]\n")) | 524 "bug(z) [ Linux ] userscripts/first-test.html [ Failure ]\n")) |
| 525 | 525 |
| 526 def test_rebaseline_updates_expectations_file_all_platforms(self): | 526 def test_rebaseline_updates_expectations_file_all_platforms(self): |
| 527 self._write(self.mac_expectations_path, "Bug(x) userscripts/first-test.h
tml [ Failure ]\n") | 527 self._write(self.mac_expectations_path, "Bug(x) userscripts/first-test.h
tml [ Failure ]\n") |
| 528 self._write("userscripts/first-test.html", "Dummy test contents") | 528 self._write("userscripts/first-test.html", "Dummy test contents") |
| 529 self._setup_mock_build_data() | 529 self._setup_mock_build_data() |
| 530 self.command._rebaseline( | 530 self.command.rebaseline( |
| 531 self.options(), | 531 self.options(), |
| 532 {"userscripts/first-test.html": {Build("MOCK Mac10.11"): ["txt", "pn
g"]}}) | 532 {"userscripts/first-test.html": {Build("MOCK Mac10.11"): ["txt", "pn
g"]}}) |
| 533 new_expectations = self._read(self.mac_expectations_path) | 533 new_expectations = self._read(self.mac_expectations_path) |
| 534 self.assertMultiLineEqual( | 534 self.assertMultiLineEqual( |
| 535 new_expectations, "Bug(x) [ Linux Mac10.10 Win ] userscripts/first-t
est.html [ Failure ]\n") | 535 new_expectations, "Bug(x) [ Linux Mac10.10 Win ] userscripts/first-t
est.html [ Failure ]\n") |
| 536 | 536 |
| 537 def test_rebaseline_handles_platform_skips(self): | 537 def test_rebaseline_handles_platform_skips(self): |
| 538 # This test is just like test_rebaseline_updates_expectations_file_all_p
latforms(), | 538 # This test is just like test_rebaseline_updates_expectations_file_all_p
latforms(), |
| 539 # except that if a particular port happens to SKIP a test in an override
s file, | 539 # except that if a particular port happens to SKIP a test in an override
s file, |
| 540 # we count that as passing, and do not think that we still need to rebas
eline it. | 540 # we count that as passing, and do not think that we still need to rebas
eline it. |
| 541 self._write(self.mac_expectations_path, "Bug(x) userscripts/first-test.h
tml [ Failure ]\n") | 541 self._write(self.mac_expectations_path, "Bug(x) userscripts/first-test.h
tml [ Failure ]\n") |
| 542 self._write("NeverFixTests", "Bug(y) [ Android ] userscripts [ WontFix ]
\n") | 542 self._write("NeverFixTests", "Bug(y) [ Android ] userscripts [ WontFix ]
\n") |
| 543 self._write("userscripts/first-test.html", "Dummy test contents") | 543 self._write("userscripts/first-test.html", "Dummy test contents") |
| 544 self._setup_mock_build_data() | 544 self._setup_mock_build_data() |
| 545 | 545 |
| 546 self.command._rebaseline( | 546 self.command.rebaseline( |
| 547 self.options(), | 547 self.options(), |
| 548 {"userscripts/first-test.html": {Build("MOCK Mac10.11"): ["txt", "pn
g"]}}) | 548 {"userscripts/first-test.html": {Build("MOCK Mac10.11"): ["txt", "pn
g"]}}) |
| 549 | 549 |
| 550 new_expectations = self._read(self.mac_expectations_path) | 550 new_expectations = self._read(self.mac_expectations_path) |
| 551 self.assertMultiLineEqual( | 551 self.assertMultiLineEqual( |
| 552 new_expectations, "Bug(x) [ Linux Mac10.10 Win ] userscripts/first-t
est.html [ Failure ]\n") | 552 new_expectations, "Bug(x) [ Linux Mac10.10 Win ] userscripts/first-t
est.html [ Failure ]\n") |
| 553 | 553 |
| 554 def test_rebaseline_handles_skips_in_file(self): | 554 def test_rebaseline_handles_skips_in_file(self): |
| 555 # This test is like test_Rebaseline_handles_platform_skips, except that
the | 555 # This test is like test_Rebaseline_handles_platform_skips, except that
the |
| 556 # Skip is in the same (generic) file rather than a platform file. In thi
s case, | 556 # Skip is in the same (generic) file rather than a platform file. In thi
s case, |
| 557 # the Skip line should be left unmodified. Note that the first line is n
ow | 557 # the Skip line should be left unmodified. Note that the first line is n
ow |
| 558 # qualified as "[Linux Mac Win]"; if it was unqualified, it would confli
ct with | 558 # qualified as "[Linux Mac Win]"; if it was unqualified, it would confli
ct with |
| 559 # the second line. | 559 # the second line. |
| 560 self._write(self.mac_expectations_path, | 560 self._write(self.mac_expectations_path, |
| 561 ("Bug(x) [ Linux Mac Win ] userscripts/first-test.html [ Fai
lure ]\n" | 561 ("Bug(x) [ Linux Mac Win ] userscripts/first-test.html [ Fai
lure ]\n" |
| 562 "Bug(y) [ Android ] userscripts/first-test.html [ Skip ]\n"
)) | 562 "Bug(y) [ Android ] userscripts/first-test.html [ Skip ]\n"
)) |
| 563 self._write("userscripts/first-test.html", "Dummy test contents") | 563 self._write("userscripts/first-test.html", "Dummy test contents") |
| 564 self._setup_mock_build_data() | 564 self._setup_mock_build_data() |
| 565 | 565 |
| 566 self.command._rebaseline( | 566 self.command.rebaseline( |
| 567 self.options(), | 567 self.options(), |
| 568 {"userscripts/first-test.html": {Build("MOCK Mac10.11"): ["txt", "pn
g"]}}) | 568 {"userscripts/first-test.html": {Build("MOCK Mac10.11"): ["txt", "pn
g"]}}) |
| 569 | 569 |
| 570 new_expectations = self._read(self.mac_expectations_path) | 570 new_expectations = self._read(self.mac_expectations_path) |
| 571 self.assertMultiLineEqual( | 571 self.assertMultiLineEqual( |
| 572 new_expectations, | 572 new_expectations, |
| 573 ("Bug(x) [ Linux Mac10.10 Win ] userscripts/first-test.html [ Failur
e ]\n" | 573 ("Bug(x) [ Linux Mac10.10 Win ] userscripts/first-test.html [ Failur
e ]\n" |
| 574 "Bug(y) [ Android ] userscripts/first-test.html [ Skip ]\n")) | 574 "Bug(y) [ Android ] userscripts/first-test.html [ Skip ]\n")) |
| 575 | 575 |
| 576 def test_rebaseline_handles_smoke_tests(self): | 576 def test_rebaseline_handles_smoke_tests(self): |
| 577 # This test is just like test_rebaseline_handles_platform_skips, except
that we check for | 577 # This test is just like test_rebaseline_handles_platform_skips, except
that we check for |
| 578 # a test not being in the SmokeTests file, instead of using overrides fi
les. | 578 # a test not being in the SmokeTests file, instead of using overrides fi
les. |
| 579 # If a test is not part of the smoke tests, we count that as passing on
ports that only | 579 # If a test is not part of the smoke tests, we count that as passing on
ports that only |
| 580 # run smoke tests, and do not think that we still need to rebaseline it. | 580 # run smoke tests, and do not think that we still need to rebaseline it. |
| 581 self._write(self.mac_expectations_path, "Bug(x) userscripts/first-test.h
tml [ Failure ]\n") | 581 self._write(self.mac_expectations_path, "Bug(x) userscripts/first-test.h
tml [ Failure ]\n") |
| 582 self._write("SmokeTests", "fast/html/article-element.html") | 582 self._write("SmokeTests", "fast/html/article-element.html") |
| 583 self._write("userscripts/first-test.html", "Dummy test contents") | 583 self._write("userscripts/first-test.html", "Dummy test contents") |
| 584 self._setup_mock_build_data() | 584 self._setup_mock_build_data() |
| 585 | 585 |
| 586 self.command._rebaseline( | 586 self.command.rebaseline( |
| 587 self.options(), | 587 self.options(), |
| 588 {"userscripts/first-test.html": {Build("MOCK Mac10.11"): ["txt", "pn
g"]}}) | 588 {"userscripts/first-test.html": {Build("MOCK Mac10.11"): ["txt", "pn
g"]}}) |
| 589 | 589 |
| 590 new_expectations = self._read(self.mac_expectations_path) | 590 new_expectations = self._read(self.mac_expectations_path) |
| 591 self.assertMultiLineEqual( | 591 self.assertMultiLineEqual( |
| 592 new_expectations, "Bug(x) [ Linux Mac10.10 Win ] userscripts/first-t
est.html [ Failure ]\n") | 592 new_expectations, "Bug(x) [ Linux Mac10.10 Win ] userscripts/first-t
est.html [ Failure ]\n") |
| 593 | 593 |
| 594 | 594 |
| 595 class TestRebaseline(BaseTestCase): | 595 class TestRebaseline(BaseTestCase): |
| 596 # This command shares most of its logic with RebaselineJson, so these tests
just test what is different. | 596 # This command shares most of its logic with RebaselineJson, so these tests
just test what is different. |
| (...skipping 330 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 927 for cmd_line, cwd in commands: | 927 for cmd_line, cwd in commands: |
| 928 out = self.run_command(cmd_line, cwd=cwd) | 928 out = self.run_command(cmd_line, cwd=cwd) |
| 929 if 'rebaseline-test-internal' in cmd_line: | 929 if 'rebaseline-test-internal' in cmd_line: |
| 930 out = '{"add": [], "remove-lines": [{"test": "%s", "builder": "%
s"}], "delete": []}\n' % (cmd_line[8], cmd_line[6]) | 930 out = '{"add": [], "remove-lines": [{"test": "%s", "builder": "%
s"}], "delete": []}\n' % (cmd_line[8], cmd_line[6]) |
| 931 command_outputs.append([0, out, '']) | 931 command_outputs.append([0, out, '']) |
| 932 | 932 |
| 933 new_calls = self.calls[num_previous_calls:] | 933 new_calls = self.calls[num_previous_calls:] |
| 934 self.calls = self.calls[:num_previous_calls] | 934 self.calls = self.calls[:num_previous_calls] |
| 935 self.calls.append(new_calls) | 935 self.calls.append(new_calls) |
| 936 return command_outputs | 936 return command_outputs |
| OLD | NEW |