Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(1233)

Issues Search
    My Issues | Starred     Open | Closed | All

Unified Diff: third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/update_test_expectations_unittest.py

Issue 2125633002: Fill out implementation of UpdateTestExpectations script (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@updateTestExpectations2
Patch Set: Created 4 years, 5 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
« third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/update_test_expectations.py ('K') | « third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/update_test_expectations.py ('k') | no next file » | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
Index: third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/update_test_expectations_unittest.py
diff --git a/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/update_test_expectations_unittest.py b/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/update_test_expectations_unittest.py
index 655d6a4b9301680fc5f4afe50bfdf8ab05df3e02..c87d95d0bac174730882b77682f1ed481ab092d1 100644
--- a/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/update_test_expectations_unittest.py
+++ b/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/update_test_expectations_unittest.py
@@ -250,6 +250,85 @@ class UpdateTestExpectationsTest(unittest.TestCase):
self._assert_expectations_match(
updated_expectations, test_expectations_before)
+ def test_all_failure_types(self):
+ """Tests that all failure types are treated as failure."""
+ test_expectations_before = (
+ """Bug(test) test/a.html [ Failure Pass ]
+ Bug(test) test/b.html [ Failure Pass ]
+ Bug(test) test/c.html [ Failure Pass ]
+ Bug(test) test/d.html [ Failure Pass ]
+ # Remove these two since CRASH and TIMEOUT aren't considered
+ # Failure.
+ Bug(test) test/e.html [ Failure Pass ]
+ Bug(test) test/f.html [ Failure Pass ]""")
+
+ self._define_builders({
+ "WebKit Linux": {
+ "port_name": "linux-precise",
+ "specifiers": ['Precise', 'Release']
+ },
+ })
+ self._port._all_build_types = ('release',)
+ self._port._all_systems = (('precise', 'x86_64'),)
+
+ self._parse_expectations(test_expectations_before)
+ self._expectation_factory._all_results_by_builder = {
+ 'WebKit Linux': {
+ "test/a.html": ["PASS", "IMAGE"],
+ "test/b.html": ["PASS", "TEXT"],
+ "test/c.html": ["PASS", "IMAGE+TEXT"],
+ "test/d.html": ["PASS", "AUDIO"],
+ "test/e.html": ["PASS", "CRASH"],
+ "test/f.html": ["PASS", "TIMEOUT"],
+ }
+ }
+ updated_expectations = (
+ self._flake_remover.get_updated_test_expectations())
+ self._assert_expectations_match(updated_expectations, (
+ """Bug(test) test/a.html [ Failure Pass ]
+ Bug(test) test/b.html [ Failure Pass ]
+ Bug(test) test/c.html [ Failure Pass ]
+ Bug(test) test/d.html [ Failure Pass ]"""))
+
+ def test_basic_one_builder(self):
+ """Tests basic functionality with a single builder.
+
+ Test that flaky expectations with results from a single bot showing the
+ expected failure isn't occuring should be removed. Results with failures
+ of the expected type shouldn't be removed but other kinds of failures
+ allow removal.
+ """
+ test_expectations_before = (
+ """# Remove this since it's passing all runs.
+ Bug(test) test/a.html [ Failure Pass ]
+ # Remove this since, although there's a failure, it's not a timeout.
+ Bug(test) test/b.html [ Pass Timeout ]
+ # Keep since we have both crashes and passes.
+ Bug(test) test/c.html [ Crash Pass ]""")
+
+ self._define_builders({
+ "WebKit Linux": {
+ "port_name": "linux-precise",
+ "specifiers": ['Precise', 'Release']
+ },
+ })
+ self._port._all_build_types = ('release',)
+ self._port._all_systems = (('precise', 'x86_64'),)
+
+ self._parse_expectations(test_expectations_before)
+ self._expectation_factory._all_results_by_builder = {
+ 'WebKit Linux': {
+ "test/a.html": ["PASS", "PASS", "PASS"],
+ "test/b.html": ["PASS", "IMAGE", "PASS"],
+ "test/c.html": ["PASS", "CRASH", "PASS"],
+ }
+ }
+ updated_expectations = (
+ self._flake_remover.get_updated_test_expectations())
+ self._assert_expectations_match(updated_expectations, (
+ """# Keep since we have both crashes and passes.
+ Bug(test) test/c.html [ Crash Pass ]"""))
+
def test_all_failure_case(self):
"""Tests that results with all failures are not treated as non-flaky."""
test_expectations_before = (
@@ -300,6 +379,502 @@ class UpdateTestExpectationsTest(unittest.TestCase):
self._flake_remover.get_updated_test_expectations())
self._assert_expectations_match(updated_expectations, "")
+ def test_basic_multiple_builders(self):
+ """Tests basic functionality with multiple builders."""
+ test_expectations_before = (
+ """# Remove since it's passing on both builders.
+ Bug(test) test/a.html [ Failure Pass ]
+ # Keep since it's failing on the Mac builder.
+ Bug(test) test/b.html [ Failure Pass ]
+ # Keep since it's failing on the Linux builder.
+ Bug(test) test/c.html [ Failure Pass ]""")
+
+ self._define_builders({
+ "WebKit Linux": {
+ "port_name": "linux-precise",
+ "specifiers": ['Precise', 'Release']
+ },
+ "WebKit Mac10.10": {
+ "port_name": "mac-mac10.10",
+ "specifiers": ['Mac10.10', 'Release']
+ },
+ })
+
+ self._port._all_build_types = ('release',)
+ self._port._all_systems = (('mac10.10', 'x86'),
+ ('precise', 'x86_64'))
+
+ self._parse_expectations(test_expectations_before)
+ self._expectation_factory._all_results_by_builder = {
+ 'WebKit Linux': {
+ "test/a.html": ["PASS", "PASS", "PASS"],
+ "test/b.html": ["PASS", "PASS", "PASS"],
+ "test/c.html": ["AUDIO", "AUDIO", "AUDIO"],
+ },
+ 'WebKit Mac10.10': {
+ "test/a.html": ["PASS", "PASS", "PASS"],
+ "test/b.html": ["PASS", "PASS", "IMAGE"],
+ "test/c.html": ["PASS", "PASS", "PASS"],
+ },
+ }
+ updated_expectations = (
+ self._flake_remover.get_updated_test_expectations())
+ self._assert_expectations_match(updated_expectations, (
+ """# Keep since it's failing on the Mac builder.
+ Bug(test) test/b.html [ Failure Pass ]
+ # Keep since it's failing on the Linux builder.
+ Bug(test) test/c.html [ Failure Pass ]"""))
+
+ def test_multiple_builders_and_platform_specifiers(self):
+ """Tests correct operation with platform specifiers."""
+ test_expectations_before = (
+ """# Keep since it's failing on Mac results.
+ Bug(test) [ Mac ] test/a.html [ Failure Pass ]
+ # Keep since it's failing on the Windows builder.
+ Bug(test) [ Linux Win ] test/b.html [ Failure Pass ]
+ # Remove since it's passing on both Linux and Windows builders.
+ Bug(test) [ Linux Win ] test/c.html [ Failure Pass ]
+ # Remove since it's passing on Mac results
+ Bug(test) [ Mac ] test/d.html [ Failure Pass ]""")
+
+ self._define_builders({
+ "WebKit Win7": {
+ "port_name": "win-win7",
+ "specifiers": ['Win7', 'Release']
+ },
+ "WebKit Linux": {
+ "port_name": "linux-precise",
+ "specifiers": ['Precise', 'Release']
+ },
+ "WebKit Mac10.10": {
+ "port_name": "mac-mac10.10",
+ "specifiers": ['Mac10.10', 'Release']
+ },
+ })
+ self._port._all_build_types = ('release',)
+ self._port._all_systems = (('mac10.10', 'x86'),
+ ('win7', 'x86'),
+ ('precise', 'x86_64'))
+
+ self._parse_expectations(test_expectations_before)
+ self._expectation_factory._all_results_by_builder = {
+ 'WebKit Linux': {
+ "test/a.html": ["PASS", "PASS", "PASS"],
+ "test/b.html": ["PASS", "PASS", "PASS"],
+ "test/c.html": ["PASS", "PASS", "PASS"],
+ "test/d.html": ["IMAGE", "PASS", "PASS"],
+ },
+ 'WebKit Mac10.10': {
+ "test/a.html": ["PASS", "PASS", "IMAGE"],
+ "test/b.html": ["PASS", "IMAGE", "PASS"],
+ "test/c.html": ["PASS", "IMAGE", "PASS"],
+ "test/d.html": ["PASS", "PASS", "PASS"],
+ },
+ 'WebKit Win7': {
+ "test/a.html": ["PASS", "PASS", "PASS"],
+ "test/b.html": ["IMAGE", "PASS", "PASS"],
+ "test/c.html": ["PASS", "PASS", "PASS"],
+ "test/d.html": ["IMAGE", "PASS", "PASS"],
+ },
+ }
+ updated_expectations = (
+ self._flake_remover.get_updated_test_expectations())
+ self._assert_expectations_match(updated_expectations, (
+ """# Keep since it's failing on Mac results.
+ Bug(test) [ Mac ] test/a.html [ Failure Pass ]
+ # Keep since it's failing on the Windows builder.
+ Bug(test) [ Linux Win ] test/b.html [ Failure Pass ]"""))
+
+ def test_debug_release_specifiers(self):
+ """Tests correct operation of Debug/Release specifiers."""
+ test_expectations_before = (
+ """# Keep since it fails in debug.
+ Bug(test) [ Linux ] test/a.html [ Failure Pass ]
+ # Remove since the failure is in Release, Debug is all PASS.
+ Bug(test) [ Debug ] test/b.html [ Failure Pass ]
+ # Keep since there's a failure in Linux Release.
+ Bug(test) [ Release ] test/c.html [ Failure Pass ]
+ # Remove since the Release Linux builder is all passing.
+ Bug(test) [ Release Linux ] test/d.html [ Failure Pass ]
+ # Remove since all the Linux builders PASS.
+ Bug(test) [ Linux ] test/e.html [ Failure Pass ]""")
+
+ self._define_builders({
+ "WebKit Win7": {
+ "port_name": "win-win7",
+ "specifiers": ['Win7', 'Release']
+ },
+ "WebKit Win7 (dbg)": {
+ "port_name": "win-win7",
+ "specifiers": ['Win7', 'Debug']
+ },
+ "WebKit Linux": {
+ "port_name": "linux-precise",
+ "specifiers": ['Precise', 'Release']
+ },
+ "WebKit Linux (dbg)": {
+ "port_name": "linux-precise",
+ "specifiers": ['Precise', 'Debug']
+ },
+ })
+ self._port._all_build_types = ('release', 'debug')
+ self._port._all_systems = (('win7', 'x86'),
+ ('precise', 'x86_64'))
+
+ self._parse_expectations(test_expectations_before)
+ self._expectation_factory._all_results_by_builder = {
+ 'WebKit Linux': {
+ "test/a.html": ["PASS", "PASS", "PASS"],
+ "test/b.html": ["PASS", "IMAGE", "PASS"],
+ "test/c.html": ["PASS", "IMAGE", "PASS"],
+ "test/d.html": ["PASS", "PASS", "PASS"],
+ "test/e.html": ["PASS", "PASS", "PASS"],
+ },
+ 'WebKit Linux (dbg)': {
+ "test/a.html": ["PASS", "IMAGE", "PASS"],
+ "test/b.html": ["PASS", "PASS", "PASS"],
+ "test/c.html": ["PASS", "PASS", "PASS"],
+ "test/d.html": ["IMAGE", "PASS", "PASS"],
+ "test/e.html": ["PASS", "PASS", "PASS"],
+ },
+ 'WebKit Win7 (dbg)': {
+ "test/a.html": ["PASS", "PASS", "PASS"],
+ "test/b.html": ["PASS", "PASS", "PASS"],
+ "test/c.html": ["PASS", "PASS", "PASS"],
+ "test/d.html": ["PASS", "IMAGE", "PASS"],
+ "test/e.html": ["PASS", "PASS", "PASS"],
+ },
+ 'WebKit Win7': {
+ "test/a.html": ["PASS", "PASS", "PASS"],
+ "test/b.html": ["PASS", "PASS", "IMAGE"],
+ "test/c.html": ["PASS", "PASS", "PASS"],
+ "test/d.html": ["PASS", "IMAGE", "PASS"],
+ "test/e.html": ["PASS", "PASS", "PASS"],
+ },
+ }
+ updated_expectations = (
+ self._flake_remover.get_updated_test_expectations())
+ self._assert_expectations_match(updated_expectations, (
+ """# Keep since it fails in debug.
+ Bug(test) [ Linux ] test/a.html [ Failure Pass ]
+ # Keep since there's a failure in Linux Release.
+ Bug(test) [ Release ] test/c.html [ Failure Pass ]"""))
+
+ def test_preserve_comments_and_whitespace(self):
+ """Tests that comments and whitespace are preserved appropriately.
+
+ Comments and whitespace should be kept unless all the tests grouped
+ below a comment are removed. In that case the comment block should also
+ be removed.
+
+ Ex:
+ # This comment applies to the below tests.
+ Bug(test) test/a.html [ Failure Pass ]
+ Bug(test) test/b.html [ Failure Pass ]
+
+ # <some prose>
+
+ # This is another comment.
+ Bug(test) test/c.html [ Failure Pass ]
+
+ Assuming we removed a.html and c.html we get:
+ # This comment applies to the below tests.
+ Bug(test) test/b.html [ Failure Pass ]
+
+ # <some prose>
+ """
+ test_expectations_before = """
+ # Comment A - Keep since these aren't part of any test.
+ # Comment B - Keep since these aren't part of any test.
+
+ # Comment C - Remove since it's a block belonging to a
+ # Comment D - and a is removed.
+ Bug(test) test/a.html [ Failure Pass ]
+ # Comment E - Keep since it's below a.
+
+
+ # Comment F - Keep since only b is removed
+ Bug(test) test/b.html [ Failure Pass ]
+ Bug(test) test/c.html [ Failure Pass ]
+
+ # Comment G - Should be removed since both d and e will be removed.
+ Bug(test) test/d.html [ Failure Pass ]
+ Bug(test) test/e.html [ Failure Pass ]"""
+
+ self._define_builders({
+ "WebKit Linux": {
+ "port_name": "linux-precise",
+ "specifiers": ['Precise', 'Release']
+ },
+ })
+ self._port._all_build_types = ('release',)
+ self._port._all_systems = (('precise', 'x86_64'),)
+
+ self._parse_expectations(test_expectations_before)
+ self._expectation_factory._all_results_by_builder = {
+ 'WebKit Linux': {
+ "test/a.html": ["PASS", "PASS", "PASS"],
+ "test/b.html": ["PASS", "PASS", "PASS"],
+ "test/c.html": ["PASS", "IMAGE", "PASS"],
+ "test/d.html": ["PASS", "PASS", "PASS"],
+ "test/e.html": ["PASS", "PASS", "PASS"],
+ }
+ }
+ updated_expectations = (
+ self._flake_remover.get_updated_test_expectations())
+ self._assert_expectations_match(updated_expectations, (
+ """
+ # Comment A - Keep since these aren't part of any test.
+ # Comment B - Keep since these aren't part of any test.
+ # Comment E - Keep since it's below a.
+
+
+ # Comment F - Keep since only b is removed
+ Bug(test) test/c.html [ Failure Pass ]"""))
+
+ def test_no_results_on_builders(self):
+ """Tests that we remove a line that has no results on the builders.
+
+ A test that has no results returned from the builders means that all
+ runs passed or were skipped. A Skip expectation in TestExpectations
+ shouldn't be removed but otherwise the test is passing.
+ """
+ test_expectations_before = """
+ # A Skip expectation probably won't have any results but we
+ # shouldn't consider those passing so this line should remain.
+ Bug(test) test/a.html [ Skip ]
+ # This line shouldn't be removed either since it's not flaky.
+ Bug(test) test/b.html [ Failure Timeout ]
+ # The lines below should be removed since they're flaky but all runs
+ # are passing.
+ Bug(test) test/c.html [ Failure Pass ]
+ Bug(test) test/d.html [ Pass Timeout ]
+ Bug(test) test/e.html [ Crash Pass ]"""
+
+ self._define_builders({
+ "WebKit Linux": {
+ "port_name": "linux-precise",
+ "specifiers": ['Precise', 'Release']
+ },
+ })
+ self._port._all_build_types = ('release',)
+ self._port._all_systems = (('precise', 'x86_64'),)
+
+ self._parse_expectations(test_expectations_before)
+ self._expectation_factory._all_results_by_builder = {
+ 'WebKit Linux': {
+ }
+ }
+ updated_expectations = (
+ self._flake_remover.get_updated_test_expectations())
+ self._assert_expectations_match(updated_expectations, """
+ # A Skip expectation probably won't have any results but we
+ # shouldn't consider those passing so this line should remain.
+ Bug(test) test/a.html [ Skip ]
+ # This line shouldn't be removed either since it's not flaky.
+ Bug(test) test/b.html [ Failure Timeout ]""")
+
+ def test_log_missing_builders(self):
+ """Tests that we emit the appropriate error for a missing builder.
+
+ If a TestExpectation has a matching configuration what we can't resolve
+ to a builder we should emit an Error.
+ """
+
+ test_expectations_before = """
+ Bug(test) [ Win ] test/a.html [ Failure Pass ]
+ Bug(test) [ Linux ] test/b.html [ Failure Pass ]
+ # This one shouldn't emit an error since it's not flaky, we don't
+ # have to check the builder results.
+ Bug(test) test/c.html [ Failure ]
+ Bug(test) test/d.html [ Failure Pass ]
+ # This one shouldn't emit an error since it will only match the
+ # existing Linux Release configuration
+ Bug(test) [ Linux Release ] test/e.html [ Failure Pass ]"""
+
+ self._define_builders({
+ "WebKit Linux": {
+ "port_name": "linux-precise",
+ "specifiers": ['Precise', 'Release']
+ },
+ })
+
+ # Three errors should be emitted:
+ # (1) There's no Windows builders so a.html will emit an error on the
+ # first missing one.
+ # (2) There's no Linux debug builder so b.html will emit an error.
+ # (3) c.html is missing will match both the Windows and Linux dbg
+ # builders which are missing so it'll emit an error on the first one.
+ self._port._all_build_types = ('release', 'debug')
+ self._port._all_systems = (('win7', 'x86'),
+ ('precise', 'x86_64'))
+
+ self._parse_expectations(test_expectations_before)
+ self._expectation_factory._all_results_by_builder = {
+ 'WebKit Linux': {
+ "test/a.html": ["PASS", "PASS", "PASS"],
+ "test/b.html": ["PASS", "PASS", "PASS"],
+ "test/c.html": ["PASS", "IMAGE", "PASS"],
+ "test/d.html": ["PASS", "PASS", "PASS"],
+ "test/e.html": ["PASS", "IMAGE", "PASS"],
+ }
+ }
+
+ updated_expectations = (
+ self._flake_remover.get_updated_test_expectations())
+ expected_errors = '\n'.join([
+ 'Failed to get builder for config [win7, x86, release]',
+ 'Failed to get builder for config [precise, x86_64, debug]',
+ 'Failed to get builder for config [win7, x86, release]',
+ ''])
+ self.assertEqual(self._log_output.getvalue(), expected_errors)
+
+ # Also make sure we didn't remove any lines if some builders were
+ # missing.
+ self._assert_expectations_match(
+ updated_expectations, test_expectations_before)
+
+ def test_log_missing_results(self):
+ """Tests that we emit the appropriate error for missing results.
+
+ If the results dictionary we download from the builders is missing the
+ results from one of the builders we matched we should have logged an
+ error.
+ """
+ test_expectations_before = """
+ Bug(test) [ Linux ] test/a.html [ Failure Pass ]
+ # This line won't emit an error since the Linux Release results
+ # exist.
+ Bug(test) [ Linux Release ] test/b.html [ Failure Pass ]
+ Bug(test) [ Release ] test/c.html [ Failure Pass ]
+ # This line is not flaky so we shouldn't even check the results.
+ Bug(test) [ Linux ] test/d.html [ Failure ]"""
+
+ self._define_builders({
+ "WebKit Linux": {
+ "port_name": "linux-precise",
+ "specifiers": ['Precise', 'Release']
+ },
+ "WebKit Linux (dbg)": {
+ "port_name": "linux-precise",
+ "specifiers": ['Precise', 'Debug']
+ },
+ "WebKit Win7": {
+ "port_name": "win-win7",
+ "specifiers": ['Win7', 'Release']
+ },
+ "WebKit Win7 (dbg)": {
+ "port_name": "win-win7",
+ "specifiers": ['Win7', 'Debug']
+ },
+ })
+
+ # Two warnings and two errors should be emitted:
+ # (1) A warning since the results don't contain anything for the Linux
+ # (dbg) builder
+ # (2) A warning since the results don't contain anything for the Win
+ # release builder
+ # (3) The first line needs and is missing results for Linux (dbg).
+ # (4) The third line needs and is missing results for Win Release.
+ self._port._all_build_types = ('release', 'debug')
+ self._port._all_systems = (('win7', 'x86'),
+ ('precise', 'x86_64'))
+
+ self._parse_expectations(test_expectations_before)
+ self._expectation_factory._all_results_by_builder = {
+ 'WebKit Linux': {
+ "test/a.html": ["PASS", "PASS", "PASS"],
+ "test/b.html": ["PASS", "IMAGE", "PASS"],
+ "test/c.html": ["PASS", "PASS", "PASS"],
+ "test/d.html": ["PASS", "PASS", "PASS"],
+ },
+ 'WebKit Win7 (dbg)': {
+ "test/a.html": ["PASS", "PASS", "PASS"],
+ "test/b.html": ["PASS", "PASS", "PASS"],
+ "test/c.html": ["PASS", "PASS", "PASS"],
+ "test/d.html": ["PASS", "PASS", "PASS"],
+ },
+ }
+
+ updated_expectations = (
+ self._flake_remover.get_updated_test_expectations())
+ expected_errors = '\n'.join([
+ 'Downloaded results are missing results for builder "WebKit Linux (dbg)"',
+ 'Downloaded results are missing results for builder "WebKit Win7"',
+ 'Failed to find results for builder "WebKit Linux (dbg)"',
+ 'Failed to find results for builder "WebKit Win7"',
+ ''])
+ self.assertEqual(self._log_output.getvalue(), expected_errors)
+
+ # Also make sure we didn't remove any lines if some builders were
+ # missing.
+ self._assert_expectations_match(
+ updated_expectations, test_expectations_before)
+
+ def test_harness_updates_file(self):
+ """Tests that the call harness updates the TestExpectations file.
+ """
+
+ self._define_builders({
+ "WebKit Linux": {
+ "port_name": "linux-precise",
+ "specifiers": ['Precise', 'Release']
+ },
+ "WebKit Linux (dbg)": {
+ "port_name": "linux-precise",
+ "specifiers": ['Precise', 'Debug']
+ },
+ })
+
+ # Setup the mock host and port.
+ host = MockHost()
+ host.port_factory = FakePortFactory(host)
+ host.port_factory._all_build_types = ('release', 'debug')
+ host.port_factory._all_systems = (('precise', 'x86_64'),)
+
+ # Write out a fake TestExpectations file.
+ test_expectation_path = (
+ host.port_factory.get().path_to_generic_test_expectations_file())
+ test_expectations = """
+ # Remove since passing on both bots.
+ Bug(test) [ Linux ] test/a.html [ Failure Pass ]
+ # Keep since there's a failure on release bot.
+ Bug(test) [ Linux Release ] test/b.html [ Failure Pass ]
+ # Remove since it's passing on both builders.
+ Bug(test) test/c.html [ Failure Pass ]
+ # Keep since there's a failure on debug bot.
+ Bug(test) [ Linux ] test/d.html [ Failure ]"""
+ files = {
+ test_expectation_path: test_expectations
+ }
+ host.filesystem = MockFileSystem(files)
+ self._write_tests_into_filesystem(host.filesystem)
+
+ # Write out the fake builder bot results.
+ expectation_factory = FakeBotTestExpectationsFactory()
+ expectation_factory._all_results_by_builder = {
+ 'WebKit Linux': {
+ "test/a.html": ["PASS", "PASS", "PASS"],
+ "test/b.html": ["PASS", "IMAGE", "PASS"],
+ "test/c.html": ["PASS", "PASS", "PASS"],
+ "test/d.html": ["PASS", "PASS", "PASS"],
+ },
+ 'WebKit Linux (dbg)': {
+ "test/a.html": ["PASS", "PASS", "PASS"],
+ "test/b.html": ["PASS", "PASS", "PASS"],
+ "test/c.html": ["PASS", "PASS", "PASS"],
+ "test/d.html": ["IMAGE", "PASS", "PASS"],
+ },
+ }
+
+ main(host, expectation_factory, [])
+
+ self.assertEqual(host.filesystem.files[test_expectation_path], (
+ """ # Keep since there's a failure on release bot.
+ Bug(test) [ Linux Release ] test/b.html [ Failure Pass ]
+ # Keep since there's a failure on debug bot.
+ Bug(test) [ Linux ] test/d.html [ Failure ]"""))
+
def test_harness_no_expectations(self):
"""Tests that a warning is outputted if the TestExpectations file
doesn't exist.
@@ -328,3 +903,55 @@ class UpdateTestExpectationsTest(unittest.TestCase):
test_expectation_path + "\n")
self.assertEqual(self._log_output.getvalue(), expected_warning)
self.assertFalse(host.filesystem.isfile(test_expectation_path))
+
+ def test_harness_remove_all(self):
+ """Tests that removing all expectations doesn't delete the file.
+
+ Make sure we're prepared for the day when we exterminated flakes.
+ """
+
+ self._define_builders({
+ "WebKit Linux": {
+ "port_name": "linux-precise",
+ "specifiers": ['Precise', 'Release']
+ },
+ "WebKit Linux (dbg)": {
+ "port_name": "linux-precise",
+ "specifiers": ['Precise', 'Debug']
+ },
+ })
+
+ # Setup the mock host and port.
+ host = MockHost()
+ host.port_factory = FakePortFactory(host)
+ host.port_factory._all_build_types = ('release', 'debug')
+ host.port_factory._all_systems = (('precise', 'x86_64'),)
+
+ # Write out a fake TestExpectations file.
+ test_expectation_path = (
+ host.port_factory.get().path_to_generic_test_expectations_file())
+ test_expectations = """
+ # Remove since passing on both bots.
+ Bug(test) [ Linux ] test/a.html [ Failure Pass ]"""
+
+ files = {
+ test_expectation_path: test_expectations
+ }
+ host.filesystem = MockFileSystem(files)
+ self._write_tests_into_filesystem(host.filesystem)
+
+ # Write out the fake builder bot results.
+ expectation_factory = FakeBotTestExpectationsFactory()
+ expectation_factory._all_results_by_builder = {
+ 'WebKit Linux': {
+ "test/a.html": ["PASS", "PASS", "PASS"],
+ },
+ 'WebKit Linux (dbg)': {
+ "test/a.html": ["PASS", "PASS", "PASS"],
+ },
+ }
+
+ main(host, expectation_factory, [])
+
+ self.assertTrue(host.filesystem.isfile(test_expectation_path))
+ self.assertEqual(host.filesystem.files[test_expectation_path], '')
« third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/update_test_expectations.py ('K') | « third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/update_test_expectations.py ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698