| OLD | NEW |
| 1 # Copyright 2016 The Chromium Authors. All rights reserved. | 1 # Copyright 2016 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 import optparse | 5 import optparse |
| 6 import unittest | 6 import unittest |
| 7 | 7 |
| 8 from webkitpy.common.net.buildbot import Build | 8 from webkitpy.common.net.buildbot import Build |
| 9 from webkitpy.common.net.layouttestresults import LayoutTestResults | 9 from webkitpy.common.net.layouttestresults import LayoutTestResults |
| 10 from webkitpy.common.system.executive_mock import MockExecutive, MockExecutive2 | 10 from webkitpy.common.system.executive_mock import MockExecutive, MockExecutive2 |
| (...skipping 402 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 413 "first-test.html": { | 413 "first-test.html": { |
| 414 "expected": "NEEDSREBASELINE", | 414 "expected": "NEEDSREBASELINE", |
| 415 "actual": "PASS" | 415 "actual": "PASS" |
| 416 } | 416 } |
| 417 } | 417 } |
| 418 } | 418 } |
| 419 })) | 419 })) |
| 420 | 420 |
| 421 self._write(self.mac_expectations_path, "Bug(x) userscripts/first-test.h
tml [ Failure ]\n") | 421 self._write(self.mac_expectations_path, "Bug(x) userscripts/first-test.h
tml [ Failure ]\n") |
| 422 self._write("userscripts/first-test.html", "Dummy test contents") | 422 self._write("userscripts/first-test.html", "Dummy test contents") |
| 423 self.command._rebaseline(self.options(), {"userscripts/first-test.html":
{Build("MOCK Win7"): ["txt", "png"]}}) | 423 self.command.rebaseline(self.options(), {"userscripts/first-test.html":
{Build("MOCK Win7"): ["txt", "png"]}}) |
| 424 | 424 |
| 425 self.assertEqual(self.tool.executive.calls, []) | 425 self.assertEqual(self.tool.executive.calls, []) |
| 426 | 426 |
| 427 def test_rebaseline_all(self): | 427 def test_rebaseline_all(self): |
| 428 self._setup_mock_build_data() | 428 self._setup_mock_build_data() |
| 429 | 429 |
| 430 self._write("userscripts/first-test.html", "Dummy test contents") | 430 self._write("userscripts/first-test.html", "Dummy test contents") |
| 431 self.command._rebaseline(self.options(), {"userscripts/first-test.html":
{Build("MOCK Win7"): ["txt", "png"]}}) | 431 self.command.rebaseline(self.options(), {"userscripts/first-test.html":
{Build("MOCK Win7"): ["txt", "png"]}}) |
| 432 | 432 |
| 433 # Note that we have one run_in_parallel() call followed by a run_command
() | 433 # Note that we have one run_in_parallel() call followed by a run_command
() |
| 434 self.assertEqual( | 434 self.assertEqual( |
| 435 self.tool.executive.calls, | 435 self.tool.executive.calls, |
| 436 [ | 436 [ |
| 437 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt,png', | 437 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt,png', |
| 438 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--verbose']], | 438 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--verbose']], |
| 439 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 't
xt,png', | 439 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 't
xt,png', |
| 440 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--verbose']], | 440 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--verbose']], |
| 441 [['python', 'echo', 'optimize-baselines', '--no-modify-scm', '--
suffixes', 'txt,png', | 441 [['python', 'echo', 'optimize-baselines', '--no-modify-scm', '--
suffixes', 'txt,png', |
| 442 'userscripts/first-test.html', '--verbose']] | 442 'userscripts/first-test.html', '--verbose']] |
| 443 ]) | 443 ]) |
| 444 | 444 |
| 445 def test_rebaseline_debug(self): | 445 def test_rebaseline_debug(self): |
| 446 self._setup_mock_build_data() | 446 self._setup_mock_build_data() |
| 447 | 447 |
| 448 self._write("userscripts/first-test.html", "Dummy test contents") | 448 self._write("userscripts/first-test.html", "Dummy test contents") |
| 449 self.command._rebaseline(self.options(), {"userscripts/first-test.html":
{Build("MOCK Win7 (dbg)"): ["txt", "png"]}}) | 449 self.command.rebaseline(self.options(), {"userscripts/first-test.html":
{Build("MOCK Win7 (dbg)"): ["txt", "png"]}}) |
| 450 | 450 |
| 451 # Note that we have one run_in_parallel() call followed by a run_command
() | 451 # Note that we have one run_in_parallel() call followed by a run_command
() |
| 452 self.assertEqual( | 452 self.assertEqual( |
| 453 self.tool.executive.calls, | 453 self.tool.executive.calls, |
| 454 [ | 454 [ |
| 455 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt,png', | 455 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt,png', |
| 456 '--builder', 'MOCK Win7 (dbg)', '--test', 'userscripts/first-t
est.html', '--verbose']], | 456 '--builder', 'MOCK Win7 (dbg)', '--test', 'userscripts/first-t
est.html', '--verbose']], |
| 457 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 't
xt,png', '--builder', | 457 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 't
xt,png', '--builder', |
| 458 'MOCK Win7 (dbg)', '--test', 'userscripts/first-test.html', '-
-verbose']], | 458 'MOCK Win7 (dbg)', '--test', 'userscripts/first-test.html', '-
-verbose']], |
| 459 [['python', 'echo', 'optimize-baselines', '--no-modify-scm', '--
suffixes', 'txt,png', | 459 [['python', 'echo', 'optimize-baselines', '--no-modify-scm', '--
suffixes', 'txt,png', |
| 460 'userscripts/first-test.html', '--verbose']] | 460 'userscripts/first-test.html', '--verbose']] |
| 461 ]) | 461 ]) |
| 462 | 462 |
| 463 def test_no_optimize(self): | 463 def test_no_optimize(self): |
| 464 self._setup_mock_build_data() | 464 self._setup_mock_build_data() |
| 465 self._write("userscripts/first-test.html", "Dummy test contents") | 465 self._write("userscripts/first-test.html", "Dummy test contents") |
| 466 self.command._rebaseline( | 466 self.command.rebaseline( |
| 467 self.options(optimize=False), | 467 self.options(optimize=False), |
| 468 {"userscripts/first-test.html": {Build("MOCK Win7"): ["txt", "png"]}
}) | 468 {"userscripts/first-test.html": {Build("MOCK Win7"): ["txt", "png"]}
}) |
| 469 | 469 |
| 470 # Note that we have only one run_in_parallel() call | 470 # Note that we have only one run_in_parallel() call |
| 471 self.assertEqual( | 471 self.assertEqual( |
| 472 self.tool.executive.calls, | 472 self.tool.executive.calls, |
| 473 [ | 473 [ |
| 474 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt,png', | 474 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt,png', |
| 475 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--verbose']], | 475 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--verbose']], |
| 476 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 't
xt,png', | 476 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 't
xt,png', |
| 477 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--verbose']] | 477 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--verbose']] |
| 478 ]) | 478 ]) |
| 479 | 479 |
| 480 def test_results_directory(self): | 480 def test_results_directory(self): |
| 481 self._setup_mock_build_data() | 481 self._setup_mock_build_data() |
| 482 self._write("userscripts/first-test.html", "Dummy test contents") | 482 self._write("userscripts/first-test.html", "Dummy test contents") |
| 483 self.command._rebaseline( | 483 self.command.rebaseline( |
| 484 self.options(optimize=False, results_directory='/tmp'), | 484 self.options(optimize=False, results_directory='/tmp'), |
| 485 {"userscripts/first-test.html": {Build("MOCK Win7"): ["txt", "png"]}
}) | 485 {"userscripts/first-test.html": {Build("MOCK Win7"): ["txt", "png"]}
}) |
| 486 | 486 |
| 487 # Note that we have only one run_in_parallel() call | 487 # Note that we have only one run_in_parallel() call |
| 488 self.assertEqual( | 488 self.assertEqual( |
| 489 self.tool.executive.calls, | 489 self.tool.executive.calls, |
| 490 [ | 490 [ |
| 491 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt,png', | 491 [['python', 'echo', 'copy-existing-baselines-internal', '--suffi
xes', 'txt,png', |
| 492 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--results-directory', '/tmp', '--verbose']], | 492 '--builder', 'MOCK Win7', '--test', 'userscripts/first-test.ht
ml', '--results-directory', '/tmp', '--verbose']], |
| 493 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 't
xt,png', | 493 [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 't
xt,png', |
| (...skipping 21 matching lines...) Expand all Loading... |
| 515 }) | 515 }) |
| 516 | 516 |
| 517 def test_rebaseline_updates_expectations_file(self): | 517 def test_rebaseline_updates_expectations_file(self): |
| 518 self._write( | 518 self._write( |
| 519 self.mac_expectations_path, | 519 self.mac_expectations_path, |
| 520 ("Bug(x) [ Mac ] userscripts/first-test.html [ Failure ]\n" | 520 ("Bug(x) [ Mac ] userscripts/first-test.html [ Failure ]\n" |
| 521 "bug(z) [ Linux ] userscripts/first-test.html [ Failure ]\n")) | 521 "bug(z) [ Linux ] userscripts/first-test.html [ Failure ]\n")) |
| 522 self._write("userscripts/first-test.html", "Dummy test contents") | 522 self._write("userscripts/first-test.html", "Dummy test contents") |
| 523 self._setup_mock_build_data() | 523 self._setup_mock_build_data() |
| 524 | 524 |
| 525 self.command._rebaseline( | 525 self.command.rebaseline( |
| 526 self.options(), | 526 self.options(), |
| 527 {"userscripts/first-test.html": {Build("MOCK Mac10.11"): ["txt", "pn
g"]}}) | 527 {"userscripts/first-test.html": {Build("MOCK Mac10.11"): ["txt", "pn
g"]}}) |
| 528 | 528 |
| 529 new_expectations = self._read(self.mac_expectations_path) | 529 new_expectations = self._read(self.mac_expectations_path) |
| 530 self.assertMultiLineEqual( | 530 self.assertMultiLineEqual( |
| 531 new_expectations, | 531 new_expectations, |
| 532 ("Bug(x) [ Mac10.10 ] userscripts/first-test.html [ Failure ]\n" | 532 ("Bug(x) [ Mac10.10 ] userscripts/first-test.html [ Failure ]\n" |
| 533 "bug(z) [ Linux ] userscripts/first-test.html [ Failure ]\n")) | 533 "bug(z) [ Linux ] userscripts/first-test.html [ Failure ]\n")) |
| 534 | 534 |
| 535 def test_rebaseline_updates_expectations_file_all_platforms(self): | 535 def test_rebaseline_updates_expectations_file_all_platforms(self): |
| 536 self._write(self.mac_expectations_path, "Bug(x) userscripts/first-test.h
tml [ Failure ]\n") | 536 self._write(self.mac_expectations_path, "Bug(x) userscripts/first-test.h
tml [ Failure ]\n") |
| 537 self._write("userscripts/first-test.html", "Dummy test contents") | 537 self._write("userscripts/first-test.html", "Dummy test contents") |
| 538 self._setup_mock_build_data() | 538 self._setup_mock_build_data() |
| 539 self.command._rebaseline( | 539 self.command.rebaseline( |
| 540 self.options(), | 540 self.options(), |
| 541 {"userscripts/first-test.html": {Build("MOCK Mac10.11"): ["txt", "pn
g"]}}) | 541 {"userscripts/first-test.html": {Build("MOCK Mac10.11"): ["txt", "pn
g"]}}) |
| 542 new_expectations = self._read(self.mac_expectations_path) | 542 new_expectations = self._read(self.mac_expectations_path) |
| 543 self.assertMultiLineEqual( | 543 self.assertMultiLineEqual( |
| 544 new_expectations, "Bug(x) [ Linux Mac10.10 Win ] userscripts/first-t
est.html [ Failure ]\n") | 544 new_expectations, "Bug(x) [ Linux Mac10.10 Win ] userscripts/first-t
est.html [ Failure ]\n") |
| 545 | 545 |
| 546 def test_rebaseline_handles_platform_skips(self): | 546 def test_rebaseline_handles_platform_skips(self): |
| 547 # This test is just like test_rebaseline_updates_expectations_file_all_p
latforms(), | 547 # This test is just like test_rebaseline_updates_expectations_file_all_p
latforms(), |
| 548 # except that if a particular port happens to SKIP a test in an override
s file, | 548 # except that if a particular port happens to SKIP a test in an override
s file, |
| 549 # we count that as passing, and do not think that we still need to rebas
eline it. | 549 # we count that as passing, and do not think that we still need to rebas
eline it. |
| 550 self._write(self.mac_expectations_path, "Bug(x) userscripts/first-test.h
tml [ Failure ]\n") | 550 self._write(self.mac_expectations_path, "Bug(x) userscripts/first-test.h
tml [ Failure ]\n") |
| 551 self._write("NeverFixTests", "Bug(y) [ Android ] userscripts [ WontFix ]
\n") | 551 self._write("NeverFixTests", "Bug(y) [ Android ] userscripts [ WontFix ]
\n") |
| 552 self._write("userscripts/first-test.html", "Dummy test contents") | 552 self._write("userscripts/first-test.html", "Dummy test contents") |
| 553 self._setup_mock_build_data() | 553 self._setup_mock_build_data() |
| 554 | 554 |
| 555 self.command._rebaseline( | 555 self.command.rebaseline( |
| 556 self.options(), | 556 self.options(), |
| 557 {"userscripts/first-test.html": {Build("MOCK Mac10.11"): ["txt", "pn
g"]}}) | 557 {"userscripts/first-test.html": {Build("MOCK Mac10.11"): ["txt", "pn
g"]}}) |
| 558 | 558 |
| 559 new_expectations = self._read(self.mac_expectations_path) | 559 new_expectations = self._read(self.mac_expectations_path) |
| 560 self.assertMultiLineEqual( | 560 self.assertMultiLineEqual( |
| 561 new_expectations, "Bug(x) [ Linux Mac10.10 Win ] userscripts/first-t
est.html [ Failure ]\n") | 561 new_expectations, "Bug(x) [ Linux Mac10.10 Win ] userscripts/first-t
est.html [ Failure ]\n") |
| 562 | 562 |
| 563 def test_rebaseline_handles_skips_in_file(self): | 563 def test_rebaseline_handles_skips_in_file(self): |
| 564 # This test is like test_Rebaseline_handles_platform_skips, except that
the | 564 # This test is like test_Rebaseline_handles_platform_skips, except that
the |
| 565 # Skip is in the same (generic) file rather than a platform file. In thi
s case, | 565 # Skip is in the same (generic) file rather than a platform file. In thi
s case, |
| 566 # the Skip line should be left unmodified. Note that the first line is n
ow | 566 # the Skip line should be left unmodified. Note that the first line is n
ow |
| 567 # qualified as "[Linux Mac Win]"; if it was unqualified, it would confli
ct with | 567 # qualified as "[Linux Mac Win]"; if it was unqualified, it would confli
ct with |
| 568 # the second line. | 568 # the second line. |
| 569 self._write(self.mac_expectations_path, | 569 self._write(self.mac_expectations_path, |
| 570 ("Bug(x) [ Linux Mac Win ] userscripts/first-test.html [ Fai
lure ]\n" | 570 ("Bug(x) [ Linux Mac Win ] userscripts/first-test.html [ Fai
lure ]\n" |
| 571 "Bug(y) [ Android ] userscripts/first-test.html [ Skip ]\n"
)) | 571 "Bug(y) [ Android ] userscripts/first-test.html [ Skip ]\n"
)) |
| 572 self._write("userscripts/first-test.html", "Dummy test contents") | 572 self._write("userscripts/first-test.html", "Dummy test contents") |
| 573 self._setup_mock_build_data() | 573 self._setup_mock_build_data() |
| 574 | 574 |
| 575 self.command._rebaseline( | 575 self.command.rebaseline( |
| 576 self.options(), | 576 self.options(), |
| 577 {"userscripts/first-test.html": {Build("MOCK Mac10.11"): ["txt", "pn
g"]}}) | 577 {"userscripts/first-test.html": {Build("MOCK Mac10.11"): ["txt", "pn
g"]}}) |
| 578 | 578 |
| 579 new_expectations = self._read(self.mac_expectations_path) | 579 new_expectations = self._read(self.mac_expectations_path) |
| 580 self.assertMultiLineEqual( | 580 self.assertMultiLineEqual( |
| 581 new_expectations, | 581 new_expectations, |
| 582 ("Bug(x) [ Linux Mac10.10 Win ] userscripts/first-test.html [ Failur
e ]\n" | 582 ("Bug(x) [ Linux Mac10.10 Win ] userscripts/first-test.html [ Failur
e ]\n" |
| 583 "Bug(y) [ Android ] userscripts/first-test.html [ Skip ]\n")) | 583 "Bug(y) [ Android ] userscripts/first-test.html [ Skip ]\n")) |
| 584 | 584 |
| 585 def test_rebaseline_handles_smoke_tests(self): | 585 def test_rebaseline_handles_smoke_tests(self): |
| 586 # This test is just like test_rebaseline_handles_platform_skips, except
that we check for | 586 # This test is just like test_rebaseline_handles_platform_skips, except
that we check for |
| 587 # a test not being in the SmokeTests file, instead of using overrides fi
les. | 587 # a test not being in the SmokeTests file, instead of using overrides fi
les. |
| 588 # If a test is not part of the smoke tests, we count that as passing on
ports that only | 588 # If a test is not part of the smoke tests, we count that as passing on
ports that only |
| 589 # run smoke tests, and do not think that we still need to rebaseline it. | 589 # run smoke tests, and do not think that we still need to rebaseline it. |
| 590 self._write(self.mac_expectations_path, "Bug(x) userscripts/first-test.h
tml [ Failure ]\n") | 590 self._write(self.mac_expectations_path, "Bug(x) userscripts/first-test.h
tml [ Failure ]\n") |
| 591 self._write("SmokeTests", "fast/html/article-element.html") | 591 self._write("SmokeTests", "fast/html/article-element.html") |
| 592 self._write("userscripts/first-test.html", "Dummy test contents") | 592 self._write("userscripts/first-test.html", "Dummy test contents") |
| 593 self._setup_mock_build_data() | 593 self._setup_mock_build_data() |
| 594 | 594 |
| 595 self.command._rebaseline( | 595 self.command.rebaseline( |
| 596 self.options(), | 596 self.options(), |
| 597 {"userscripts/first-test.html": {Build("MOCK Mac10.11"): ["txt", "pn
g"]}}) | 597 {"userscripts/first-test.html": {Build("MOCK Mac10.11"): ["txt", "pn
g"]}}) |
| 598 | 598 |
| 599 new_expectations = self._read(self.mac_expectations_path) | 599 new_expectations = self._read(self.mac_expectations_path) |
| 600 self.assertMultiLineEqual( | 600 self.assertMultiLineEqual( |
| 601 new_expectations, "Bug(x) [ Linux Mac10.10 Win ] userscripts/first-t
est.html [ Failure ]\n") | 601 new_expectations, "Bug(x) [ Linux Mac10.10 Win ] userscripts/first-t
est.html [ Failure ]\n") |
| 602 | 602 |
| 603 | 603 |
| 604 class TestRebaseline(BaseTestCase): | 604 class TestRebaseline(BaseTestCase): |
| 605 # This command shares most of its logic with RebaselineJson, so these tests
just test what is different. | 605 # This command shares most of its logic with RebaselineJson, so these tests
just test what is different. |
| (...skipping 330 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 936 for cmd_line, cwd in commands: | 936 for cmd_line, cwd in commands: |
| 937 out = self.run_command(cmd_line, cwd=cwd) | 937 out = self.run_command(cmd_line, cwd=cwd) |
| 938 if 'rebaseline-test-internal' in cmd_line: | 938 if 'rebaseline-test-internal' in cmd_line: |
| 939 out = '{"add": [], "remove-lines": [{"test": "%s", "builder": "%
s"}], "delete": []}\n' % (cmd_line[8], cmd_line[6]) | 939 out = '{"add": [], "remove-lines": [{"test": "%s", "builder": "%
s"}], "delete": []}\n' % (cmd_line[8], cmd_line[6]) |
| 940 command_outputs.append([0, out, '']) | 940 command_outputs.append([0, out, '']) |
| 941 | 941 |
| 942 new_calls = self.calls[num_previous_calls:] | 942 new_calls = self.calls[num_previous_calls:] |
| 943 self.calls = self.calls[:num_previous_calls] | 943 self.calls = self.calls[:num_previous_calls] |
| 944 self.calls.append(new_calls) | 944 self.calls.append(new_calls) |
| 945 return command_outputs | 945 return command_outputs |
| OLD | NEW |